def test_get_connectors_for_user(self):
connector = Connector.objects.create(name='MySql',
dialect='mysql',
settings=json.dumps([{
"name":
"url",
"value":
"mysql://*****:*****@hue:3306/hue"
}]))
# Could leverate update_app_permissions() instead of adding manually the permission but this is more lightweight for now
conn_perm = HuePermission.objects.create(app=connector.name,
action='access',
description='',
connector=connector)
GroupPermission.objects.create(group=self.user.groups.first(),
hue_permission=conn_perm)
try:
assert_true(self.user.get_permissions())
connectors = _get_installed_connectors(user=self.user)
assert_true(connectors, connectors)
connectors = _get_installed_connectors(user=self.alone_user)
assert_false(connectors, connectors)
finally:
connector.delete()
def test_get_connectors_for_user(self):
with patch(
'desktop.lib.connectors.models.CONNECTORS.get') as CONNECTORS:
CONNECTORS.return_value = {
'mysql-1':
Mock(
NICE_NAME=Mock(get=Mock(return_value='MySql')),
DIALECT=Mock(get=Mock(return_value='mysql')),
INTERFACE=Mock(get=Mock(return_value='sqlalchemy')),
SETTINGS=Mock(get=Mock(
return_value=[{
"name": "url",
"value": "mysql://*****:*****@hue:3306/hue"
}])),
)
}
update_app_permissions()
connectors = _get_installed_connectors(user=self.user)
assert_true(connectors, connectors)
connectors = _get_installed_connectors(user=self.alone_user)
assert_false(connectors, connectors)
def test_get_installed_editor_connectors():
with patch('desktop.lib.connectors.api.CONNECTORS.get') as CONNECTORS:
CONNECTORS.return_value = {
'mysql-1':
Mock(
NICE_NAME=Mock(get=Mock(return_value='MySql')),
DIALECT=Mock(get=Mock(return_value='mysql')),
INTERFACE=Mock(get=Mock(return_value='sqlalchemy')),
SETTINGS=Mock(get=Mock(
return_value=[{
"name": "url",
"value": "mysql://*****:*****@hue:3306/hue"
}])),
)
}
connectors = _get_installed_connectors()
editor_category = [
category for category in connectors
if category['category'] == 'editor'
]
assert_true(len(editor_category), connectors)
assert_equal(1, len(editor_category), editor_category)
def test_config_validator(self, has_connectors):
with patch(
'desktop.lib.connectors.models.CONNECTORS.get') as CONNECTORS:
CONNECTORS.return_value = {
'hive-1':
Mock(
NICE_NAME=Mock(get=Mock(return_value='Hive')),
DIALECT=Mock(get=Mock(return_value='hive')),
INTERFACE=Mock(get=Mock(return_value='hiveserver2')),
SETTINGS=Mock(get=Mock(return_value=[{
"name": "server_host",
"value": "gethue"
}, {
"name": "server_port",
"value": "10000"
}])),
)
}
update_app_permissions()
connectors = _get_installed_connectors(user=self.user)
assert_true(connectors, connectors)
warnings = config_validator(user=self.user)
assert_true(warnings, warnings)
assert_equal('hive-1', warnings[0][0])
assert_true(
'Testing the connector connection failed' in warnings[0][1],
warnings)
def get_ordered_interpreters(user=None):
if has_connectors():
from desktop.lib.connectors.api import _get_installed_connectors
interpreters = [
_connector_to_iterpreter(connector)
for connector in _get_installed_connectors(
categories=['editor', 'catalogs'], user=user)
]
else:
if not INTERPRETERS.get():
_default_interpreters(user)
interpreters = INTERPRETERS.get()
user_apps = appmanager.get_apps_dict(user)
user_interpreters = []
for interpreter in interpreters:
if check_permissions(user, interpreter, user_apps=user_apps):
pass # Not allowed
else:
user_interpreters.append(interpreter)
interpreters_shown_on_wheel = _remove_duplications(
INTERPRETERS_SHOWN_ON_WHEEL.get())
unknown_interpreters = set(interpreters_shown_on_wheel) - set(
user_interpreters)
if unknown_interpreters:
# Just filtering it out might be better than failing for this user
raise ValueError(
"Interpreters from interpreters_shown_on_wheel is not in the list of Interpreters %s"
% unknown_interpreters)
reordered_interpreters = interpreters_shown_on_wheel + [
i
for i in user_interpreters if i not in interpreters_shown_on_wheel
]
interpreters = [{
'name': interpreters[i].NAME.get(),
'type': i,
'interface': interpreters[i].INTERFACE.get(),
'options': interpreters[i].OPTIONS.get()
} for i in reordered_interpreters]
return [{
"name": i.get('nice_name', i['name']),
"type": i['type'],
"interface": i['interface'],
"options": i['options'],
'dialect': i.get('dialect', i['name']).lower(),
'dialect_properties': i.get('dialect_properties'),
'category': i.get('category', 'editor'),
"is_sql": i.get('is_sql') or \
i['interface'] in ["hiveserver2", "rdbms", "jdbc", "solr", "sqlalchemy", "ksql", "flink"] or \
i['type'] == 'sql',
"is_catalog": i['interface'] in ["hms",],
}
for i in interpreters
]
def has_hdfs_enabled():
if has_connectors():
from desktop.lib.connectors.api import _get_installed_connectors
return any([
connector for connector in _get_installed_connectors()
if connector['dialect'] == 'hdfs'
])
else:
return list(HDFS_CLUSTERS.keys())
def get_ordered_interpreters(user=None):
if not INTERPRETERS.get():
_default_interpreters(user)
interpreters = INTERPRETERS.get()
interpreters_shown_on_wheel = _remove_duplications(INTERPRETERS_SHOWN_ON_WHEEL.get())
user_apps = appmanager.get_apps_dict(user)
user_interpreters = []
for interpreter in interpreters:
if check_permissions(user, interpreter, user_apps=user_apps):
pass # Not allowed
else:
user_interpreters.append(interpreter)
unknown_interpreters = set(interpreters_shown_on_wheel) - set(user_interpreters)
if unknown_interpreters:
raise ValueError("Interpreters from interpreters_shown_on_wheel is not in the list of Interpreters %s" % unknown_interpreters)
if has_connectors():
from desktop.lib.connectors.api import _get_installed_connectors
reordered_interpreters = [{
'name': connector['nice_name'],
'type': connector['name'],
'dialect': connector['dialect'],
'category': connector['category'],
'is_sql': connector.get('is_sql', False),
'interface': connector['interface'],
'options': {setting['name']: setting['value'] for setting in connector['settings']}
} for connector in _get_installed_connectors(categories=['editor', 'catalogs'])
]
else:
reordered_interpreters = interpreters_shown_on_wheel + [i for i in user_interpreters if i not in interpreters_shown_on_wheel]
reordered_interpreters = [{
'name': interpreters[i].NAME.get(),
'type': i,
'interface': interpreters[i].INTERFACE.get(),
'options': interpreters[i].OPTIONS.get()
} for i in reordered_interpreters
]
return [{
"name": i.get('nice_name', i['name']),
"type": i['type'],
"interface": i['interface'],
"options": i['options'],
'dialect': i.get('dialect', i['name']).lower(),
'category': i.get('category', 'editor'),
"is_sql": i.get('is_sql') or i['interface'] in ["hiveserver2", "rdbms", "jdbc", "solr", "sqlalchemy"],
"is_catalog": i['interface'] in ["hms",],
}
for i in reordered_interpreters
]
def test_config_validator(self, has_connectors):
with patch('desktop.lib.connectors.api._get_installed_connectors'
) as _get_installed_connectors:
with patch(
'notebook.conf._excute_test_query') as _excute_test_query:
_get_installed_connectors.return_value = [{
'nice_name': 'Hive',
'name': 'hive-1',
'dialect': 'hive',
'category': 'editor',
'interface': 'hiveserver2',
'settings': {},
'dialect_properties': {
'sql_identifier_quote': '`',
'is_sql': True
},
}]
_excute_test_query.return_value = Mock(
content=json.dumps({'status': 0}))
connectors = _get_installed_connectors(user=self.user)
assert_true(connectors, connectors)
warnings = config_validator(user=self.user)
assert_false(warnings, warnings)
_excute_test_query.side_effect = Exception('')
connectors = _get_installed_connectors(user=self.user)
assert_true(connectors, connectors)
warnings = config_validator(user=self.user)
assert_true(warnings, warnings)
assert_equal('Hive - hive (hive-1)', warnings[0][0])
assert_true(
'Testing the connector connection failed'
in warnings[0][1], warnings)
def get_all_hdfs():
global FS_CACHE
if FS_CACHE:
return FS_CACHE
FS_CACHE = {}
if has_connectors():
for connector in _get_installed_connectors(category='browsers',
dialect='hdfs',
interface='rest'):
settings = {
setting['name']: setting['value']
for setting in connector['settings']
}
FS_CACHE[connector['name']] = webhdfs.WebHdfs(
url=settings['server_url'],
fs_defaultfs=settings['default_fs'])
else:
for identifier in list(conf.HDFS_CLUSTERS.keys()):
FS_CACHE[identifier] = _make_filesystem(identifier)
return FS_CACHE
def test_get_installed_editor_connectors(self):
with patch('desktop.lib.connectors.models.Connector.objects.all'
) as ConnectorObjectsAll:
ConnectorObjectsAll.return_value = [
Connector(name='MySql',
dialect='mysql',
settings=json.dumps([{
"name":
"url",
"value":
"mysql://*****:*****@hue:3306/hue"
}]))
]
connectors = _get_installed_connectors()
editor_category = [
category for category in connectors
if category['category'] == 'editor'
]
assert_true(editor_category, connectors)
assert_equal(1, len(editor_category), editor_category)
def get_ordered_interpreters(user=None):
global INTERPRETERS_CACHE
if has_connectors():
from desktop.lib.connectors.api import _get_installed_connectors
interpreters = [
_connector_to_interpreter(connector)
for connector in _get_installed_connectors(
categories=['editor', 'catalogs'], user=user)
]
else:
if INTERPRETERS_CACHE is None:
none_user = None # for getting full list of interpreters
if is_cm_managed():
extra_interpreters = INTERPRETERS.get(
) # Combine the other apps interpreters
_default_interpreters(none_user)
else:
extra_interpreters = {}
if not INTERPRETERS.get():
_default_interpreters(none_user)
INTERPRETERS_CACHE = INTERPRETERS.get()
INTERPRETERS_CACHE.update(extra_interpreters)
user_apps = appmanager.get_apps_dict(user)
user_interpreters = []
for interpreter in INTERPRETERS_CACHE:
if check_has_missing_permission(user,
interpreter,
user_apps=user_apps):
pass # Not allowed
else:
user_interpreters.append(interpreter)
interpreters_shown_on_wheel = _remove_duplications(
INTERPRETERS_SHOWN_ON_WHEEL.get())
unknown_interpreters = set(interpreters_shown_on_wheel) - set(
user_interpreters)
if unknown_interpreters:
# Just filtering it out might be better than failing for this user
raise ValueError(
"Interpreters from interpreters_shown_on_wheel is not in the list of Interpreters %s"
% unknown_interpreters)
reordered_interpreters = interpreters_shown_on_wheel + [
i
for i in user_interpreters if i not in interpreters_shown_on_wheel
]
interpreters = [{
'name': INTERPRETERS_CACHE[i].NAME.get(),
'type': i,
'interface': INTERPRETERS_CACHE[i].INTERFACE.get(),
'options': INTERPRETERS_CACHE[i].OPTIONS.get()
} for i in reordered_interpreters]
return [{
"name": i.get('nice_name', i['name']),
'displayName': 'Unified Analytics' if ENABLE_UNIFIED_ANALYTICS.get() and i.get('dialect', i['name']).lower() == 'hive' else i.get('nice_name', i['name']),
"type": i['type'],
"interface": i['interface'],
"options": i['options'],
'dialect': i.get('dialect', i['name']).lower(),
'dialect_properties': i.get('dialect_properties') or {}, # Empty when connectors off
'category': i.get('category', 'editor'),
"is_sql": i.get('is_sql') or \
i['interface'] in ["hiveserver2", "rdbms", "jdbc", "solr", "sqlalchemy", "ksql", "flink"] or \
i['type'] == 'sql',
"is_catalog": i['interface'] in ["hms",],
}
for i in interpreters
]
