def test_get_connectors_for_user(self): connector = Connector.objects.create(name='MySql', dialect='mysql', settings=json.dumps([{ "name": "url", "value": "mysql://*****:*****@hue:3306/hue" }])) # Could leverate update_app_permissions() instead of adding manually the permission but this is more lightweight for now conn_perm = HuePermission.objects.create(app=connector.name, action='access', description='', connector=connector) GroupPermission.objects.create(group=self.user.groups.first(), hue_permission=conn_perm) try: assert_true(self.user.get_permissions()) connectors = _get_installed_connectors(user=self.user) assert_true(connectors, connectors) connectors = _get_installed_connectors(user=self.alone_user) assert_false(connectors, connectors) finally: connector.delete()
def test_get_connectors_for_user(self): with patch( 'desktop.lib.connectors.models.CONNECTORS.get') as CONNECTORS: CONNECTORS.return_value = { 'mysql-1': Mock( NICE_NAME=Mock(get=Mock(return_value='MySql')), DIALECT=Mock(get=Mock(return_value='mysql')), INTERFACE=Mock(get=Mock(return_value='sqlalchemy')), SETTINGS=Mock(get=Mock( return_value=[{ "name": "url", "value": "mysql://*****:*****@hue:3306/hue" }])), ) } update_app_permissions() connectors = _get_installed_connectors(user=self.user) assert_true(connectors, connectors) connectors = _get_installed_connectors(user=self.alone_user) assert_false(connectors, connectors)
def test_get_installed_editor_connectors(): with patch('desktop.lib.connectors.api.CONNECTORS.get') as CONNECTORS: CONNECTORS.return_value = { 'mysql-1': Mock( NICE_NAME=Mock(get=Mock(return_value='MySql')), DIALECT=Mock(get=Mock(return_value='mysql')), INTERFACE=Mock(get=Mock(return_value='sqlalchemy')), SETTINGS=Mock(get=Mock( return_value=[{ "name": "url", "value": "mysql://*****:*****@hue:3306/hue" }])), ) } connectors = _get_installed_connectors() editor_category = [ category for category in connectors if category['category'] == 'editor' ] assert_true(len(editor_category), connectors) assert_equal(1, len(editor_category), editor_category)
def test_config_validator(self, has_connectors): with patch( 'desktop.lib.connectors.models.CONNECTORS.get') as CONNECTORS: CONNECTORS.return_value = { 'hive-1': Mock( NICE_NAME=Mock(get=Mock(return_value='Hive')), DIALECT=Mock(get=Mock(return_value='hive')), INTERFACE=Mock(get=Mock(return_value='hiveserver2')), SETTINGS=Mock(get=Mock(return_value=[{ "name": "server_host", "value": "gethue" }, { "name": "server_port", "value": "10000" }])), ) } update_app_permissions() connectors = _get_installed_connectors(user=self.user) assert_true(connectors, connectors) warnings = config_validator(user=self.user) assert_true(warnings, warnings) assert_equal('hive-1', warnings[0][0]) assert_true( 'Testing the connector connection failed' in warnings[0][1], warnings)
def get_ordered_interpreters(user=None): if has_connectors(): from desktop.lib.connectors.api import _get_installed_connectors interpreters = [ _connector_to_iterpreter(connector) for connector in _get_installed_connectors( categories=['editor', 'catalogs'], user=user) ] else: if not INTERPRETERS.get(): _default_interpreters(user) interpreters = INTERPRETERS.get() user_apps = appmanager.get_apps_dict(user) user_interpreters = [] for interpreter in interpreters: if check_permissions(user, interpreter, user_apps=user_apps): pass # Not allowed else: user_interpreters.append(interpreter) interpreters_shown_on_wheel = _remove_duplications( INTERPRETERS_SHOWN_ON_WHEEL.get()) unknown_interpreters = set(interpreters_shown_on_wheel) - set( user_interpreters) if unknown_interpreters: # Just filtering it out might be better than failing for this user raise ValueError( "Interpreters from interpreters_shown_on_wheel is not in the list of Interpreters %s" % unknown_interpreters) reordered_interpreters = interpreters_shown_on_wheel + [ i for i in user_interpreters if i not in interpreters_shown_on_wheel ] interpreters = [{ 'name': interpreters[i].NAME.get(), 'type': i, 'interface': interpreters[i].INTERFACE.get(), 'options': interpreters[i].OPTIONS.get() } for i in reordered_interpreters] return [{ "name": i.get('nice_name', i['name']), "type": i['type'], "interface": i['interface'], "options": i['options'], 'dialect': i.get('dialect', i['name']).lower(), 'dialect_properties': i.get('dialect_properties'), 'category': i.get('category', 'editor'), "is_sql": i.get('is_sql') or \ i['interface'] in ["hiveserver2", "rdbms", "jdbc", "solr", "sqlalchemy", "ksql", "flink"] or \ i['type'] == 'sql', "is_catalog": i['interface'] in ["hms",], } for i in interpreters ]
def has_hdfs_enabled(): if has_connectors(): from desktop.lib.connectors.api import _get_installed_connectors return any([ connector for connector in _get_installed_connectors() if connector['dialect'] == 'hdfs' ]) else: return list(HDFS_CLUSTERS.keys())
def get_ordered_interpreters(user=None): if not INTERPRETERS.get(): _default_interpreters(user) interpreters = INTERPRETERS.get() interpreters_shown_on_wheel = _remove_duplications(INTERPRETERS_SHOWN_ON_WHEEL.get()) user_apps = appmanager.get_apps_dict(user) user_interpreters = [] for interpreter in interpreters: if check_permissions(user, interpreter, user_apps=user_apps): pass # Not allowed else: user_interpreters.append(interpreter) unknown_interpreters = set(interpreters_shown_on_wheel) - set(user_interpreters) if unknown_interpreters: raise ValueError("Interpreters from interpreters_shown_on_wheel is not in the list of Interpreters %s" % unknown_interpreters) if has_connectors(): from desktop.lib.connectors.api import _get_installed_connectors reordered_interpreters = [{ 'name': connector['nice_name'], 'type': connector['name'], 'dialect': connector['dialect'], 'category': connector['category'], 'is_sql': connector.get('is_sql', False), 'interface': connector['interface'], 'options': {setting['name']: setting['value'] for setting in connector['settings']} } for connector in _get_installed_connectors(categories=['editor', 'catalogs']) ] else: reordered_interpreters = interpreters_shown_on_wheel + [i for i in user_interpreters if i not in interpreters_shown_on_wheel] reordered_interpreters = [{ 'name': interpreters[i].NAME.get(), 'type': i, 'interface': interpreters[i].INTERFACE.get(), 'options': interpreters[i].OPTIONS.get() } for i in reordered_interpreters ] return [{ "name": i.get('nice_name', i['name']), "type": i['type'], "interface": i['interface'], "options": i['options'], 'dialect': i.get('dialect', i['name']).lower(), 'category': i.get('category', 'editor'), "is_sql": i.get('is_sql') or i['interface'] in ["hiveserver2", "rdbms", "jdbc", "solr", "sqlalchemy"], "is_catalog": i['interface'] in ["hms",], } for i in reordered_interpreters ]
def test_config_validator(self, has_connectors): with patch('desktop.lib.connectors.api._get_installed_connectors' ) as _get_installed_connectors: with patch( 'notebook.conf._excute_test_query') as _excute_test_query: _get_installed_connectors.return_value = [{ 'nice_name': 'Hive', 'name': 'hive-1', 'dialect': 'hive', 'category': 'editor', 'interface': 'hiveserver2', 'settings': {}, 'dialect_properties': { 'sql_identifier_quote': '`', 'is_sql': True }, }] _excute_test_query.return_value = Mock( content=json.dumps({'status': 0})) connectors = _get_installed_connectors(user=self.user) assert_true(connectors, connectors) warnings = config_validator(user=self.user) assert_false(warnings, warnings) _excute_test_query.side_effect = Exception('') connectors = _get_installed_connectors(user=self.user) assert_true(connectors, connectors) warnings = config_validator(user=self.user) assert_true(warnings, warnings) assert_equal('Hive - hive (hive-1)', warnings[0][0]) assert_true( 'Testing the connector connection failed' in warnings[0][1], warnings)
def get_all_hdfs(): global FS_CACHE if FS_CACHE: return FS_CACHE FS_CACHE = {} if has_connectors(): for connector in _get_installed_connectors(category='browsers', dialect='hdfs', interface='rest'): settings = { setting['name']: setting['value'] for setting in connector['settings'] } FS_CACHE[connector['name']] = webhdfs.WebHdfs( url=settings['server_url'], fs_defaultfs=settings['default_fs']) else: for identifier in list(conf.HDFS_CLUSTERS.keys()): FS_CACHE[identifier] = _make_filesystem(identifier) return FS_CACHE
def test_get_installed_editor_connectors(self): with patch('desktop.lib.connectors.models.Connector.objects.all' ) as ConnectorObjectsAll: ConnectorObjectsAll.return_value = [ Connector(name='MySql', dialect='mysql', settings=json.dumps([{ "name": "url", "value": "mysql://*****:*****@hue:3306/hue" }])) ] connectors = _get_installed_connectors() editor_category = [ category for category in connectors if category['category'] == 'editor' ] assert_true(editor_category, connectors) assert_equal(1, len(editor_category), editor_category)
def get_ordered_interpreters(user=None): global INTERPRETERS_CACHE if has_connectors(): from desktop.lib.connectors.api import _get_installed_connectors interpreters = [ _connector_to_interpreter(connector) for connector in _get_installed_connectors( categories=['editor', 'catalogs'], user=user) ] else: if INTERPRETERS_CACHE is None: none_user = None # for getting full list of interpreters if is_cm_managed(): extra_interpreters = INTERPRETERS.get( ) # Combine the other apps interpreters _default_interpreters(none_user) else: extra_interpreters = {} if not INTERPRETERS.get(): _default_interpreters(none_user) INTERPRETERS_CACHE = INTERPRETERS.get() INTERPRETERS_CACHE.update(extra_interpreters) user_apps = appmanager.get_apps_dict(user) user_interpreters = [] for interpreter in INTERPRETERS_CACHE: if check_has_missing_permission(user, interpreter, user_apps=user_apps): pass # Not allowed else: user_interpreters.append(interpreter) interpreters_shown_on_wheel = _remove_duplications( INTERPRETERS_SHOWN_ON_WHEEL.get()) unknown_interpreters = set(interpreters_shown_on_wheel) - set( user_interpreters) if unknown_interpreters: # Just filtering it out might be better than failing for this user raise ValueError( "Interpreters from interpreters_shown_on_wheel is not in the list of Interpreters %s" % unknown_interpreters) reordered_interpreters = interpreters_shown_on_wheel + [ i for i in user_interpreters if i not in interpreters_shown_on_wheel ] interpreters = [{ 'name': INTERPRETERS_CACHE[i].NAME.get(), 'type': i, 'interface': INTERPRETERS_CACHE[i].INTERFACE.get(), 'options': INTERPRETERS_CACHE[i].OPTIONS.get() } for i in reordered_interpreters] return [{ "name": i.get('nice_name', i['name']), 'displayName': 'Unified Analytics' if ENABLE_UNIFIED_ANALYTICS.get() and i.get('dialect', i['name']).lower() == 'hive' else i.get('nice_name', i['name']), "type": i['type'], "interface": i['interface'], "options": i['options'], 'dialect': i.get('dialect', i['name']).lower(), 'dialect_properties': i.get('dialect_properties') or {}, # Empty when connectors off 'category': i.get('category', 'editor'), "is_sql": i.get('is_sql') or \ i['interface'] in ["hiveserver2", "rdbms", "jdbc", "solr", "sqlalchemy", "ksql", "flink"] or \ i['type'] == 'sql', "is_catalog": i['interface'] in ["hms",], } for i in interpreters ]