def test_get_interpreters_to_show(): default_interpreters = OrderedDict(( ('hive', { 'name': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'is_sql': True, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'hive' }), ('spark', { 'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'scala' }), ('pig', { 'name': 'Pig', 'interface': 'pig', 'type': 'pig', 'is_sql': False, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'pig' }), ('java', { 'name': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'java' }) )) expected_interpreters = OrderedDict(( ('java', { 'name': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'java' }), ('pig', { 'name': 'Pig', 'interface': 'pig', 'is_sql': False, 'type': 'pig', 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'pig' }), ('hive', { 'name': 'Hive', 'interface': 'hiveserver2', 'is_sql': True, 'type': 'hive', 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'hive' }), ('spark', { 'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {}, 'is_catalog': False, 'category': 'editor', 'dialect': 'scala' }) )) try: resets = [INTERPRETERS.set_for_testing(default_interpreters), APP_BLACKLIST.set_for_testing('')] appmanager.DESKTOP_MODULES = [] appmanager.DESKTOP_APPS = None appmanager.load_apps(APP_BLACKLIST.get()) interpreters_shown_on_wheel_unset = get_ordered_interpreters() assert_equal( list(default_interpreters.values()), interpreters_shown_on_wheel_unset, 'get_interpreters_to_show should return the same as get_interpreters when interpreters_shown_on_wheel is unset. expected: %s, actual: %s' % ( list(default_interpreters.values()), interpreters_shown_on_wheel_unset ) ) resets.append(INTERPRETERS_SHOWN_ON_WHEEL.set_for_testing('java,pig')) assert_equal(list(expected_interpreters.values()), get_ordered_interpreters(), 'get_interpreters_to_show did not return interpreters in the correct order expected: %s, actual: %s' % (list(expected_interpreters.values()), get_ordered_interpreters())) finally: for reset in resets: reset() appmanager.DESKTOP_MODULES = [] appmanager.DESKTOP_APPS = None appmanager.load_apps(APP_BLACKLIST.get())
def _get_interpreter_from_dialect(dialect, user): if not dialect: interpreter = get_ordered_interpreters(user=user)[0] elif '-' in dialect: interpreter = { 'dialect': dialect.split('-')[0], 'type': dialect.split('-')[1] # Id } else: interpreter = [ i for i in get_ordered_interpreters(user=user) if i['dialect'] == dialect ][0] return interpreter
def editor(request, is_mobile=False, is_embeddable=False): editor_id = request.GET.get('editor') editor_type = request.GET.get('type', 'hive') if editor_type == 'notebook' or request.GET.get('notebook'): return notebook(request) if editor_id: # Open existing saved editor document editor_type = _get_editor_type(editor_id) template = 'editor.mako' if is_mobile: template = 'editor_m.mako' return render( template, request, { 'editor_id': editor_id or None, 'notebooks_json': '{}', 'is_embeddable': request.GET.get('is_embeddable', False), 'editor_type': editor_type, 'options_json': json.dumps({ 'languages': get_ordered_interpreters(request.user), 'mode': 'editor', 'is_optimizer_enabled': has_optimizer(), 'is_wa_enabled': has_workload_analytics(), 'is_navigator_enabled': has_catalog(request.user), 'editor_type': editor_type, 'mobile': is_mobile }) })
def get_engines(user): engines = [] apps = get_apps_dict() settings = get_properties() if 'search' in apps: engines.append({ 'name': _('index (Solr)'), 'type': 'solr', 'analytics': settings.get('solr') and settings['solr'].get('analytics'), 'nesting': settings.get('solr') and settings['solr'].get('nesting'), }) if HAS_SQL_ENABLED.get() and ('beeswax' in apps or 'rdbms' in apps): engines += [{ 'name': _('table (%s)') % interpreter['name'], 'type': interpreter['type'], 'async': interpreter['interface'] == 'hiveserver2', 'analytics': settings.get('sql') and settings['sql'].get('analytics'), 'nesting': settings.get('sql') and settings['sql'].get('nesting'), } for interpreter in get_ordered_interpreters(user) if interpreter['interface'] in ('hiveserver2', 'jdbc', 'rdbms')] return engines
def get_interpreter(connector_type, user=None): interpreter = [ interpreter for interpreter in get_ordered_interpreters(user) if connector_type == interpreter['type'] ] if not interpreter: if connector_type == 'hbase': # TODO move to connectors interpreter = [{ 'name': 'hbase', 'type': 'hbase', 'interface': 'hbase', 'options': {}, 'is_sql': False }] elif connector_type == 'kafka': interpreter = [{ 'name': 'kafka', 'type': 'kafka', 'interface': 'kafka', 'options': {}, 'is_sql': False }] elif connector_type == 'solr': interpreter = [{ 'name': 'solr', 'type': 'solr', 'interface': 'solr', 'options': {}, 'is_sql': False }] else: raise PopupException(_('Snippet type %s is not configured.') % connector_type) elif len(interpreter) > 1: raise PopupException(_('Snippet type %s matching more than one interpreter: %s') % (connector_type, len(interpreter))) return interpreter[0]
def hue(request): apps = appmanager.get_apps_dict(request.user) return render( 'hue.mako', request, { 'apps': apps, 'interpreters': get_ordered_interpreters(request.user), 'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(), 'leaflet': { 'layer': desktop.conf.LEAFLET_TILE_LAYER.get(), 'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get() }, 'is_demo': desktop.conf.DEMO_ENABLED.get(), 'banner_message': get_banner_message(request), 'cluster_config': ClusterConfig(request.user), 'user_preferences': dict((x.key, x.value) for x in UserPreferences.objects.filter(user=request.user)) })
def browse(request, database, table, partition_spec=None): snippet = {'type': request.POST.get('sourceType', 'hive')} statement = get_api(request, snippet).get_browse_query(snippet, database, table, partition_spec) editor_type = snippet['type'] if request.method == 'POST': notebook = make_notebook(name='Execute and watch', editor_type=editor_type, statement=statement, status='ready-execute', is_task=True) return JsonResponse(notebook.execute(request, batch=False)) else: editor = make_notebook(name='Browse', editor_type=editor_type, statement=statement, status='ready-execute') return render( 'editor.mako', request, { 'notebooks_json': json.dumps([editor.get_data()]), 'options_json': json.dumps({ 'languages': get_ordered_interpreters(request.user), 'mode': 'editor', 'editor_type': editor_type }), 'editor_type': editor_type, })
def editor(request, is_mobile=False, is_embeddable=False): editor_id = request.GET.get("editor") editor_type = request.GET.get("type", "hive") if editor_id: # Open existing saved editor document document = Document2.objects.get(id=editor_id) editor_type = document.type.rsplit("-", 1)[-1] template = "editor.mako" if is_mobile: template = "editor_m.mako" if is_embeddable: template = "editor_embeddable.mako" return render( template, request, { "editor_id": editor_id or None, "notebooks_json": "{}", "editor_type": editor_type, "options_json": json.dumps( { "languages": get_ordered_interpreters(request.user), "mode": "editor", "is_optimizer_enabled": has_optimizer(), "is_navigator_enabled": has_navigator(request.user), "editor_type": editor_type, "mobile": is_mobile, } ), }, )
def jdbc_db_list(request): format_ = {'data': [], 'status': 1} interpreters = get_ordered_interpreters(request.user) format_['data'] = [{'value': key['name'], 'name': key['name']} for key in interpreters if key['interface'] == 'jdbc'] format_['status'] = 0 return JsonResponse(format_)
def notebook(request, is_embeddable=False): notebook_id = request.GET.get("notebook") is_yarn_mode = False try: from spark.conf import LIVY_SERVER_SESSION_KIND is_yarn_mode = LIVY_SERVER_SESSION_KIND.get() except: LOG.exception("Spark is not enabled") template = "notebook.mako" if is_embeddable: template = "notebook_embeddable.mako" return render( template, request, { "editor_id": notebook_id or None, "notebooks_json": "{}", "options_json": json.dumps( { "languages": get_ordered_interpreters(request.user), "session_properties": SparkApi.get_properties(), "is_optimizer_enabled": has_optimizer(), "is_navigator_enabled": has_navigator(request.user), "editor_type": "notebook", } ), "is_yarn_mode": is_yarn_mode, }, )
def hue(request): apps = appmanager.get_apps_dict(request.user) return render( 'hue.mako', request, { 'apps': apps, 'tours_and_tutorials': Settings.get_settings().tours_and_tutorials, 'interpreters': get_ordered_interpreters(request.user), 'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get(), 'leaflet': { 'layer': desktop.conf.LEAFLET_TILE_LAYER.get(), 'attribution': desktop.conf.LEAFLET_TILE_LAYER_ATTRIBUTION.get() }, 'is_demo': desktop.conf.DEMO_ENABLED.get(), 'banner_message': get_banner_message(request), 'cluster_config': ClusterConfig(request.user) })
def editor(request, is_mobile=False, is_embeddable=False): editor_id = request.GET.get('editor') editor_type = request.GET.get('type', 'hive') if editor_type == 'notebook' or request.GET.get('notebook'): return notebook(request) if editor_id: # Open existing saved editor document document = Document2.objects.get(id=editor_id) editor_type = document.type.rsplit('-', 1)[-1] template = 'editor.mako' if is_mobile: template = 'editor_m.mako' return render(template, request, { 'editor_id': editor_id or None, 'notebooks_json': '{}', 'is_embeddable': request.GET.get('is_embeddable', False), 'editor_type': editor_type, 'options_json': json.dumps({ 'languages': get_ordered_interpreters(request.user), 'mode': 'editor', 'is_optimizer_enabled': has_optimizer(), 'is_navigator_enabled': has_navigator(request.user), 'editor_type': editor_type, 'mobile': is_mobile }) })
def test_get_ordered_interpreters(self): with patch('desktop.lib.connectors.api._get_installed_connectors' ) as _get_installed_connectors: _get_installed_connectors.return_value = [{ 'nice_name': 'Hive', 'name': 'hive-1', 'dialect': 'hive', 'category': 'editor', 'interface': 'hiveserver2', 'settings': {}, 'dialect_properties': { 'sql_identifier_quote': '`', 'is_sql': True }, }] interpreters = get_ordered_interpreters(user=self.user) assert_true(interpreters, interpreters) assert_true( all([ 'dialect_properties' in interpreter for interpreter in interpreters ]), interpreters) assert_true( any([ interpreter.get('dialect_properties').get( 'sql_identifier_quote') for interpreter in interpreters ]), interpreters)
def notebook(request, is_embeddable=False): if not SHOW_NOTEBOOKS.get(): return serve_403_error(request) notebook_id = request.GET.get('notebook', request.GET.get('editor')) is_yarn_mode = False try: from spark.conf import LIVY_SERVER_SESSION_KIND is_yarn_mode = LIVY_SERVER_SESSION_KIND.get() except: LOG.exception('Spark is not enabled') return render('notebook.mako', request, { 'editor_id': notebook_id or None, 'notebooks_json': '{}', 'is_embeddable': request.GET.get('is_embeddable', False), 'options_json': json.dumps({ 'languages': get_ordered_interpreters(request.user), 'session_properties': SparkApi.get_properties(), 'is_optimizer_enabled': has_optimizer(), 'is_navigator_enabled': has_navigator(request.user), 'editor_type': 'notebook' }), 'is_yarn_mode': is_yarn_mode, })
def notebook(request, is_embeddable=False): notebook_id = request.GET.get('notebook') is_yarn_mode = False try: from spark.conf import LIVY_SERVER_SESSION_KIND is_yarn_mode = LIVY_SERVER_SESSION_KIND.get() except: LOG.exception('Spark is not enabled') template = 'notebook.mako' if is_embeddable: template = 'notebook_embeddable.mako' return render( template, request, { 'editor_id': notebook_id or None, 'notebooks_json': '{}', 'options_json': json.dumps({ 'languages': get_ordered_interpreters(request.user), 'session_properties': SparkApi.get_properties(), 'is_optimizer_enabled': has_optimizer(), 'is_navigator_enabled': has_navigator(request.user), 'editor_type': 'notebook' }), 'is_yarn_mode': is_yarn_mode, })
def install_custom_examples(): if EXAMPLES.AUTO_LOAD.get(): from desktop.auth.backend import rewrite_user from beeswax.management.commands import beeswax_install_examples from useradmin.models import install_sample_user user = rewrite_user(install_sample_user()) if has_connectors(): interpreters = [{ 'type': connector['id'], 'dialect': connector['dialect'] } for connector in _get_installed_connectors(category='editor')] else: interpreters = [ { 'type': interpreter['dialect'], 'dialect': interpreter['dialect'] } for interpreter in get_ordered_interpreters(user) # Only for hive/impala currently, would also need to port to Notebook install examples. if interpreter['dialect'] in ('hive', 'impala') ] queries = EXAMPLES.QUERIES.get() tables = EXAMPLES.TABLES.get( ) # No-op. Only for the saved query samples, not the tables currently. LOG.info( 'Installing custom examples queries: %(queries)s, tables: %(tables)s for dialects %(dialects)s ' 'belonging to user %(user)s' % { 'queries': queries, 'tables': tables, 'dialects': [interpreter['dialect'] for interpreter in interpreters], 'user': user }) result = [] for interpreter in interpreters: successes, errors = beeswax_install_examples.Command().handle( dialect=interpreter['dialect'], user=user, interpreter=interpreter, queries=queries, tables=tables, request=None) LOG.info( 'Dialect %(dialect)s installed samples: %(successes)s, %(errors)s,' % { 'dialect': interpreter['dialect'], 'successes': successes, 'errors': errors, }) result.append((successes, errors)) return result
def test_get_interpreters_to_show(): default_interpreters = OrderedDict(( ('hive', { 'name': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'options': {} }), ('spark', { 'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'options': {} }), ('pig', { 'name': 'Pig', 'interface': 'pig', 'type': 'pig', 'options': {} }), ('java', { 'name': 'Java', 'interface': 'oozie', 'type': 'java', 'options': {} }) )) expected_interpreters = OrderedDict(( ('java', { 'name': 'Java', 'interface': 'oozie', 'type': 'java', 'options': {} }), ('pig', { 'name': 'Pig', 'interface': 'pig', 'type': 'pig', 'options': {} }), ('hive', { 'name': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'options': {} }), ('spark', { 'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'options': {} }) )) try: resets = [INTERPRETERS.set_for_testing(default_interpreters)] interpreters_shown_on_wheel_unset = get_ordered_interpreters() assert_equal(default_interpreters.values(), interpreters_shown_on_wheel_unset, 'get_interpreters_to_show should return the same as get_interpreters when ' 'interpreters_shown_on_wheel is unset. expected: %s, actual: %s' % (default_interpreters.values(), interpreters_shown_on_wheel_unset)) resets.append(INTERPRETERS_SHOWN_ON_WHEEL.set_for_testing('java,pig')) assert_equal(expected_interpreters.values(), get_ordered_interpreters(), 'get_interpreters_to_show did not return interpreters in the correct order expected: %s, actual: %s' % (expected_interpreters.values(), get_ordered_interpreters())) finally: for reset in resets: reset()
def test_get_interpreters_to_show(): default_interpreters = OrderedDict(( ('hive', { 'name': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'is_sql': True, 'options': {} }), ('spark', { 'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {} }), ('pig', { 'name': 'Pig', 'interface': 'pig', 'type': 'pig', 'is_sql': False, 'options': {} }), ('java', { 'name': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {} }) )) expected_interpreters = OrderedDict(( ('java', { 'name': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {} }), ('pig', { 'name': 'Pig', 'interface': 'pig', 'is_sql': False, 'type': 'pig', 'options': {} }), ('hive', { 'name': 'Hive', 'interface': 'hiveserver2', 'is_sql': True, 'type': 'hive', 'options': {} }), ('spark', { 'name': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {} }) )) try: resets = [INTERPRETERS.set_for_testing(default_interpreters)] interpreters_shown_on_wheel_unset = get_ordered_interpreters() assert_equal(default_interpreters.values(), interpreters_shown_on_wheel_unset, 'get_interpreters_to_show should return the same as get_interpreters when ' 'interpreters_shown_on_wheel is unset. expected: %s, actual: %s' % (default_interpreters.values(), interpreters_shown_on_wheel_unset)) resets.append(INTERPRETERS_SHOWN_ON_WHEEL.set_for_testing('java,pig')) assert_equal(expected_interpreters.values(), get_ordered_interpreters(), 'get_interpreters_to_show did not return interpreters in the correct order expected: %s, actual: %s' % (expected_interpreters.values(), get_ordered_interpreters())) finally: for reset in resets: reset()
def responsive(request): apps = appmanager.get_apps_dict(request.user) return render('responsive.mako', request, { 'apps': apps, 'tours_and_tutorials': Settings.get_settings().tours_and_tutorials, 'interpreters': get_ordered_interpreters(request.user), 'is_s3_enabled': is_s3_enabled() and has_s3_access(request.user), 'is_ldap_setup': 'desktop.auth.backend.LdapBackend' in desktop.conf.AUTH.BACKEND.get() })
def get_db_component(request): format_ = {'data': [], 'status': 1, 'message': ''} db = None try: source = json.loads(request.POST.get('source', '{}')) user = User.objects.get(username=request.user) name = None if source['rdbmsMode'] == 'configRdbms': if source['rdbmsType'] != 'jdbc': query_server = rdbms.get_query_server_config(server=source['rdbmsType']) db = rdbms.get(user, query_server=query_server) else: interpreters = get_ordered_interpreters(request.user) options = {} key = [key for key in interpreters if key['name'] == source['rdbmsJdbcDriverName']] if key: options = key[0]['options'] db = Jdbc(driver_name=options['driver'], url=options['url'], username=options['user'], password=options['password']) else: name = source['rdbmsType'] if name != 'jdbc': query_server = { 'server_name': name, 'server_host': source['rdbmsHostname'], 'server_port': int(source['rdbmsPort']), 'username': source['rdbmsUsername'], 'password': source['rdbmsPassword'], 'options': {}, 'alias': name } db = rdbms.get(user, query_server=query_server) else: db = Jdbc(driver_name=source['rdbmsJdbcDriver'], url=source['rdbmsHostname'], username=source['rdbmsUsername'], password=source['rdbmsPassword']) if source['rdbmsType'] != 'jdbc': assist = Assist(db) else: assist = JdbcAssist(db) if not source['rdbmsDatabaseName']: data = assist.get_databases() elif source['rdbmsDatabaseName']: data = assist.get_tables(source['rdbmsDatabaseName']) format_['data'] = [{'name': element, 'value': element} for element in data] format_['status'] = 0 except Exception, e: message = _('Error accessing the database %s: %s') % (name, e) LOG.warn(message) format['message'] = message
def get_engines(user): engines = [{'name': _('index (Solr)'), 'type': 'solr'}] if ENABLE_SQL.get(): engines += [{ 'name': _('table (%s)') % interpreter['name'], 'type': interpreter['type'], 'async': interpreter['interface'] == 'hiveserver2' } for interpreter in get_ordered_interpreters(user) if interpreter['interface'] in ('hiveserver2', 'jdbc', 'rdbms') ] return engines
def editor(request, is_mobile=False, is_embeddable=False): editor_id = request.GET.get('editor') editor_type = request.GET.get('type', 'hive') gist_id = request.GET.get('gist') if editor_type == 'notebook' or request.GET.get('notebook'): return notebook(request) if editor_type == 'gist': gist_doc = _get_gist_document(uuid=gist_id) editor_type = gist_doc.extra if EXAMPLES.AUTO_OPEN.get() and not editor_id: sample_query = _get_dialect_example(dialect=editor_type) if sample_query: editor_id = sample_query.id if editor_id and not gist_id: # Open existing saved editor document editor_type = _get_editor_type(editor_id) template = 'editor.mako' if ENABLE_NOTEBOOK_2.get(): template = 'editor2.mako' elif is_mobile: template = 'editor_m.mako' return render( template, request, { 'editor_id': editor_id or None, 'notebooks_json': '{}', 'is_embeddable': request.GET.get('is_embeddable', False), 'editor_type': editor_type, 'options_json': json.dumps({ 'languages': get_ordered_interpreters(request.user), 'mode': 'editor', 'is_optimizer_enabled': has_optimizer(), 'is_wa_enabled': has_workload_analytics(), 'is_navigator_enabled': has_catalog(request.user), 'editor_type': editor_type, 'mobile': is_mobile }) })
def get_api(request, snippet): from notebook.connectors.dataeng import DataEngApi from notebook.connectors.hiveserver2 import HS2Api from notebook.connectors.jdbc import JdbcApi from notebook.connectors.rdbms import RdbmsApi from notebook.connectors.oozie_batch import OozieApi from notebook.connectors.solr import SolrApi from notebook.connectors.spark_shell import SparkApi from notebook.connectors.spark_batch import SparkBatchApi from notebook.connectors.text import TextApi if snippet.get('wasBatchExecuted'): return OozieApi(user=request.user, request=request) interpreter = [interpreter for interpreter in get_ordered_interpreters(request.user) if interpreter['type'] == snippet['type']] if not interpreter: raise PopupException(_('Snippet type %(type)s is not configured in hue.ini') % snippet) interpreter = interpreter[0] interface = interpreter['interface'] # Multi cluster cluster = Cluster(request.user) if cluster and cluster.get_type() == 'dataeng': interface = 'dataeng' if interface == 'hiveserver2': return HS2Api(user=request.user, request=request) elif interface == 'oozie': return OozieApi(user=request.user, request=request) elif interface == 'livy': return SparkApi(request.user) elif interface == 'livy-batch': return SparkBatchApi(request.user) elif interface == 'text' or interface == 'markdown': return TextApi(request.user) elif interface == 'rdbms': return RdbmsApi(request.user, interpreter=snippet['type']) elif interface == 'dataeng': return DataEngApi(user=request.user, request=request, cluster_name=cluster.get_interface()) elif interface == 'jdbc': return JdbcApi(request.user, interpreter=interpreter) elif interface == 'solr': return SolrApi(request.user, interpreter=interpreter) elif interface == 'pig': return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4 else: raise PopupException(_('Notebook connector interface not recognized: %s') % interface)
def _get_db(request): source = json.loads( request.POST.get('source', request.POST.get('fileFormat', '{}'))) user = User.objects.get(username=request.user) name = None if source['rdbmsMode'] == 'configRdbms': if source['rdbmsType'] != 'jdbc': query_server = rdbms.get_query_server_config( server=source['rdbmsType']) db = rdbms.get(user, query_server=query_server) else: interpreters = get_ordered_interpreters(request.user) options = {} key = [ key for key in interpreters if key['name'] == source['rdbmsJdbcDriverName'] ] if key: options = key[0]['options'] db = Jdbc(driver_name=options['driver'], url=options['url'], username=options['user'], password=options['password']) else: name = source['rdbmsType'] if name != 'jdbc': query_server = { 'server_name': name, 'server_host': source['rdbmsHostname'], 'server_port': int(source['rdbmsPort']), 'username': source['rdbmsUsername'], 'password': source['rdbmsPassword'], 'options': {}, 'alias': name } db = rdbms.get(user, query_server=query_server) else: db = Jdbc(driver_name=source['rdbmsJdbcDriver'], url=source['rdbmsHostname'], username=source['rdbmsUsername'], password=source['rdbmsPassword']) return db
def browse(request, database, table): snippet = {"type": "hive"} sql_select = get_api(request, snippet).get_select_star_query(snippet, database, table) editor_type = snippet["type"] editor = make_notebook(name="Browse", editor_type=editor_type, statement=sql_select, status="ready-execute") return render( "editor.mako", request, { "notebooks_json": json.dumps([editor.get_data()]), "options_json": json.dumps( {"languages": get_ordered_interpreters(request.user), "mode": "editor", "editor_type": editor_type} ), "editor_type": editor_type, }, )
def get_api(request, snippet): from notebook.connectors.hiveserver2 import HS2Api from notebook.connectors.jdbc import JdbcApi from notebook.connectors.rdbms import RdbmsApi from notebook.connectors.oozie_batch import OozieApi from notebook.connectors.solr import SolrApi from notebook.connectors.spark_shell import SparkApi from notebook.connectors.spark_batch import SparkBatchApi from notebook.connectors.text import TextApi if snippet.get('wasBatchExecuted'): return OozieApi(user=request.user, request=request) interpreter = [interpreter for interpreter in get_ordered_interpreters(request.user) if interpreter['type'] == snippet['type']] if not interpreter: raise PopupException(_('Snippet type %(type)s is not configured in hue.ini') % snippet) interpreter = interpreter[0] interface = interpreter['interface'] if interface == 'hiveserver2': return HS2Api(user=request.user, request=request) elif interface == 'oozie': return OozieApi(user=request.user, request=request) elif interface == 'livy': return SparkApi(request.user) elif interface == 'livy-batch': return SparkBatchApi(request.user) elif interface == 'text' or interface == 'markdown': return TextApi(request.user) elif interface == 'rdbms': return RdbmsApi(request.user, interpreter=snippet['type']) elif interface == 'jdbc': return JdbcApi(request.user, interpreter=interpreter) elif interface == 'solr': return SolrApi(request.user, interpreter=interpreter) elif interface == 'pig': return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4 else: raise PopupException(_('Notebook connector interface not recognized: %s') % interface)
def browse(request, database, table): snippet = {'type': 'hive'} sql_select = get_api(request, snippet).get_select_star_query( snippet, database, table) editor_type = snippet['type'] editor = make_notebook(name='Browse', editor_type=editor_type, statement=sql_select, status='ready-execute') return render( 'editor.mako', request, { 'notebooks_json': json.dumps([editor.get_data()]), 'options_json': json.dumps({ 'languages': get_ordered_interpreters(request.user), 'mode': 'editor', 'editor_type': editor_type }), 'editor_type': editor_type, })
# limitations under the License. # Start DBProxy server if we have some JDBC snippets from notebook.conf import get_ordered_interpreters, ENABLE_DBPROXY_SERVER def _start_livy_server(): import atexit import subprocess import sys import time p = subprocess.Popen([sys.executable, sys.argv[0], 'dbproxy_server']) def cleanup(): p.terminate() for _ in xrange(5): if p.poll() == None: time.sleep(1) else: break else: p.kill() atexit.register(cleanup) if ENABLE_DBPROXY_SERVER.get() and [interpreter for interpreter in get_ordered_interpreters() if interpreter['interface'] == 'jdbc']: _start_livy_server()
def test_get_ordered_interpreters(): default_interpreters = OrderedDict(( ('hive', { 'name': 'Hive', 'displayName': 'Hive', 'interface': 'hiveserver2', 'type': 'hive', 'is_sql': True, 'options': {}, 'dialect_properties': None, 'is_catalog': False, 'category': 'editor', 'dialect': 'hive' }), ('impala', { 'name': 'Impala', 'displayName': 'Impala', 'interface': 'hiveserver2', 'type': 'impala', 'is_sql': True, 'options': {}, 'dialect_properties': None, 'is_catalog': False, 'category': 'editor', 'dialect': 'impala' }), ('spark', { 'name': 'Scala', 'displayName': 'Scala', 'interface': 'livy', 'type': 'spark', 'is_sql': False, 'options': {}, 'dialect_properties': None, 'is_catalog': False, 'category': 'editor', 'dialect': 'scala' }), ('pig', { 'name': 'Pig', 'displayName': 'Pig', 'interface': 'pig', 'type': 'pig', 'is_sql': False, 'options': {}, 'dialect_properties': None, 'is_catalog': False, 'category': 'editor', 'dialect': 'pig' }), ('java', { 'name': 'Java', 'displayName': 'Java', 'interface': 'oozie', 'type': 'java', 'is_sql': False, 'options': {}, 'dialect_properties': None, 'is_catalog': False, 'category': 'editor', 'dialect': 'java' }) )) try: resets = [APP_BLACKLIST.set_for_testing('')] appmanager.DESKTOP_MODULES = [] appmanager.DESKTOP_APPS = None appmanager.load_apps(APP_BLACKLIST.get()) with patch('notebook.conf.is_cm_managed') as is_cm_managed: with patch('notebook.conf.appmanager.get_apps_dict') as get_apps_dict: with patch('notebook.conf.has_connectors') as has_connectors: get_apps_dict.return_value = {'hive': {}} has_connectors.return_value = False notebook.conf.INTERPRETERS_CACHE = None is_cm_managed.return_value = False # No CM --> Verbatim INTERPRETERS.set_for_testing( OrderedDict(( ('phoenix', { 'name': 'Phoenix', 'interface': 'sqlalchemy', 'dialect': 'phoenix' }),) ) ) assert_equal( [interpreter['dialect'] for interpreter in get_ordered_interpreters()], ['phoenix'] ) assert_equal( # Check twice because of cache [interpreter['dialect'] for interpreter in get_ordered_interpreters()], ['phoenix'] ) is_cm_managed.return_value = True notebook.conf.INTERPRETERS_CACHE = None # CM --> Append [] INTERPRETERS.set_for_testing( OrderedDict(() ) ) assert_equal( [interpreter['dialect'] for interpreter in get_ordered_interpreters()], ['hive'] ) assert_equal( # Check twice [interpreter['dialect'] for interpreter in get_ordered_interpreters()], ['hive'] ) notebook.conf.INTERPRETERS_CACHE = None # CM --> Append [Phoenix] INTERPRETERS.set_for_testing( OrderedDict(( ('phoenix', { 'name': 'Phoenix', 'interface': 'sqlalchemy', 'dialect': 'phoenix' }),) ) ) assert_equal( [interpreter['dialect'] for interpreter in get_ordered_interpreters()], ['hive', 'phoenix'] ) assert_equal( # Check twice [interpreter['dialect'] for interpreter in get_ordered_interpreters()], ['hive', 'phoenix'] ) finally: for reset in resets: reset() appmanager.DESKTOP_MODULES = [] appmanager.DESKTOP_APPS = None appmanager.load_apps(APP_BLACKLIST.get()) notebook.conf.INTERPRETERS_CACHE = None
from notebook.conf import get_ordered_interpreters, ENABLE_DBPROXY_SERVER def _start_livy_server(): import atexit import subprocess import sys import time p = subprocess.Popen([sys.executable, sys.argv[0], 'dbproxy_server']) def cleanup(): p.terminate() for _ in xrange(5): if p.poll() == None: time.sleep(1) else: break else: p.kill() atexit.register(cleanup) if ENABLE_DBPROXY_SERVER.get() and [ interpreter for interpreter in get_ordered_interpreters() if interpreter['interface'] == 'jdbc' ]: _start_livy_server()
def create_table_from_file(self, request, source, destination, start_time=-1, dry_run=False): if '.' in destination['name']: database, table_name = destination['name'].split('.', 1) else: database = 'default' table_name = destination['name'] final_table_name = table_name source_type = [interpreter['type'] for interpreter in get_ordered_interpreters(self.user) if interpreter['dialect'] == 'phoenix'][0] editor_type = source_type columns = destination['columns'] # Until we have proper type convertion for col in columns: if col['type'] == 'string': col['type'] = 'varchar' sql = '''CREATE TABLE IF NOT EXISTS %(table_name)s ( %(columns)s CONSTRAINT my_pk PRIMARY KEY (%(primary_keys)s) ); ''' % { 'database': database, 'table_name': table_name, 'columns': ',\n'.join([' %(name)s %(type)s' % col for col in columns]), 'primary_keys': ', '.join(destination.get('indexerPrimaryKey')) } source_path = urllib_unquote(source['path']) if source['inputFormat'] == 'file': file_obj = request.fs.open(source_path) content = file_obj.read().decode("utf-8") csvfile = string_io(content) reader = csv.reader(csvfile) else: local_file = open(source_path, 'r') reader = csv.reader(local_file) if destination['indexerRunJob']: for count, csv_row in enumerate(reader): if (source['format']['hasHeader'] and count == 0) or not csv_row: continue else: _sql = ', '.join([ "'{0}'".format(col_val) if columns[count]['type'] in ('varchar', 'timestamp') \ else '{0}'.format(col_val) for count, col_val in enumerate(csv_row)]) sql += '''\nUPSERT INTO %(table_name)s VALUES (%(csv_row)s);\n''' % { 'database': database, 'table_name': table_name, 'csv_row': _sql } if dry_run: return sql else: on_success_url = reverse('metastore:describe_table', kwargs={'database': database, 'table': final_table_name}) + \ '?source_type=' + source_type return make_notebook( name=_('Creating table %(database)s.%(table)s') % {'database': database, 'table': final_table_name}, editor_type=editor_type, statement=sql.strip(), status='ready', database=database, on_success_url=on_success_url, last_executed=start_time, is_task=True )
def get_api(request, snippet): from notebook.connectors.oozie_batch import OozieApi if snippet.get('wasBatchExecuted'): return OozieApi(user=request.user, request=request) if snippet['type'] == 'report': snippet['type'] = 'impala' interpreter = [ interpreter for interpreter in get_ordered_interpreters(request.user) if interpreter['type'] == snippet['type'] ] if not interpreter: if snippet['type'] == 'hbase': interpreter = [{ 'name': 'hbase', 'type': 'hbase', 'interface': 'hbase', 'options': {}, 'is_sql': False }] elif snippet['type'] == 'kafka': interpreter = [{ 'name': 'kafka', 'type': 'kafka', 'interface': 'kafka', 'options': {}, 'is_sql': False }] elif snippet['type'] == 'solr': interpreter = [{ 'name': 'solr', 'type': 'solr', 'interface': 'solr', 'options': {}, 'is_sql': False }] else: raise PopupException( _('Snippet type %(type)s is not configured in hue.ini') % snippet) interpreter = interpreter[0] interface = interpreter['interface'] # Multi cluster cluster = json.loads(request.POST.get('cluster', '""')) # Via Catalog API if cluster == 'undefined': cluster = None if not cluster and snippet.get('compute'): # Via notebook.ko.js cluster = snippet.get('compute').get('id') if cluster and 'crn:altus:dataware:' in cluster: interface = 'altus-adb' if cluster: LOG.info('Selected cluster %s' % cluster) if interface == 'hiveserver2': from notebook.connectors.hiveserver2 import HS2Api return HS2Api(user=request.user, request=request, cluster=cluster) elif interface == 'oozie': return OozieApi(user=request.user, request=request) elif interface == 'livy': from notebook.connectors.spark_shell import SparkApi return SparkApi(request.user) elif interface == 'livy-batch': from notebook.connectors.spark_batch import SparkBatchApi return SparkBatchApi(request.user) elif interface == 'text' or interface == 'markdown': from notebook.connectors.text import TextApi return TextApi(request.user) elif interface == 'rdbms': from notebook.connectors.rdbms import RdbmsApi return RdbmsApi(request.user, interpreter=snippet['type']) elif interface == 'altus-adb': from notebook.connectors.altus_adb import AltusAdbApi return AltusAdbApi(user=request.user, cluster_name=cluster, request=request) elif interface == 'dataeng': from notebook.connectors.dataeng import DataEngApi return DataEngApi(user=request.user, request=request, cluster_name=cluster.get('name')) elif interface == 'jdbc' or interface == 'teradata': from notebook.connectors.jdbc import JdbcApi return JdbcApi(request.user, interpreter=interpreter) elif interface == 'sqlalchemy': from notebook.connectors.sqlalchemyapi import SqlAlchemyApi return SqlAlchemyApi(request.user, interpreter=interpreter) elif interface == 'solr': from notebook.connectors.solr import SolrApi return SolrApi(request.user, interpreter=interpreter) elif interface == 'hbase': from notebook.connectors.hbase import HBaseApi return HBaseApi(request.user) elif interface == 'kafka': from notebook.connectors.kafka import KafkaApi return KafkaApi(request.user) elif interface == 'pig': return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4 else: raise PopupException( _('Notebook connector interface not recognized: %s') % interface)
def get_api(request, snippet): from notebook.connectors.oozie_batch import OozieApi if snippet.get('wasBatchExecuted'): return OozieApi(user=request.user, request=request) if snippet['type'] == 'report': snippet['type'] = 'impala' interpreter = [ interpreter for interpreter in get_ordered_interpreters(request.user) if snippet['type'] in (interpreter['type'], interpreter['interface']) ] if not interpreter: if snippet['type'] == 'hbase': interpreter = [{ 'name': 'hbase', 'type': 'hbase', 'interface': 'hbase', 'options': {}, 'is_sql': False }] elif snippet['type'] == 'kafka': interpreter = [{ 'name': 'kafka', 'type': 'kafka', 'interface': 'kafka', 'options': {}, 'is_sql': False }] elif snippet['type'] == 'solr': interpreter = [{ 'name': 'solr', 'type': 'solr', 'interface': 'solr', 'options': {}, 'is_sql': False }] elif snippet['type'] == 'custom': interpreter = [{ 'name': snippet['name'], 'type': snippet['type'], 'interface': snippet['interface'], 'options': snippet.get('options', {}), 'is_sql': False }] else: raise PopupException( _('Snippet type %(type)s is not configured.') % snippet) interpreter = interpreter[0] interface = interpreter['interface'] if CONNECTORS.IS_ENABLED.get(): cluster = { 'connector': snippet['type'], 'id': interpreter['type'], } snippet['type'] = snippet['type'].split('-', 2)[0] cluster.update(interpreter['options']) # Multi cluster elif has_multi_cluster(): cluster = json.loads(request.POST.get( 'cluster', '""')) # Via Catalog autocomplete API or Notebook create sessions if cluster == '""' or cluster == 'undefined': cluster = None if not cluster and snippet.get('compute'): # Via notebook.ko.js cluster = snippet['compute'] else: cluster = None cluster_name = cluster.get('id') if cluster else None if cluster and 'altus:dataware:k8s' in cluster_name: interface = 'hiveserver2' elif cluster and 'crn:altus:dataware:' in cluster_name: interface = 'altus-adb' elif cluster and 'crn:altus:dataeng:' in cluster_name: interface = 'dataeng' LOG.info('Selected cluster %s %s interface %s' % (cluster_name, cluster, interface)) snippet['interface'] = interface if interface.startswith('hiveserver2') or interface == 'hms': from notebook.connectors.hiveserver2 import HS2Api return HS2Api(user=request.user, request=request, cluster=cluster, interface=interface) elif interface == 'oozie': return OozieApi(user=request.user, request=request) elif interface == 'livy': from notebook.connectors.spark_shell import SparkApi return SparkApi(request.user) elif interface == 'livy-batch': from notebook.connectors.spark_batch import SparkBatchApi return SparkBatchApi(request.user) elif interface == 'text' or interface == 'markdown': from notebook.connectors.text import TextApi return TextApi(request.user) elif interface == 'rdbms': from notebook.connectors.rdbms import RdbmsApi return RdbmsApi(request.user, interpreter=snippet['type'], query_server=snippet.get('query_server')) elif interface == 'altus-adb': from notebook.connectors.altus_adb import AltusAdbApi return AltusAdbApi(user=request.user, cluster_name=cluster_name, request=request) elif interface == 'dataeng': from notebook.connectors.dataeng import DataEngApi return DataEngApi(user=request.user, request=request, cluster_name=cluster_name) elif interface == 'jdbc': if interpreter['options'] and interpreter['options'].get( 'url', '').find('teradata') >= 0: from notebook.connectors.jdbc_teradata import JdbcApiTeradata return JdbcApiTeradata(request.user, interpreter=interpreter) if interpreter['options'] and interpreter['options'].get( 'url', '').find('awsathena') >= 0: from notebook.connectors.jdbc_athena import JdbcApiAthena return JdbcApiAthena(request.user, interpreter=interpreter) elif interpreter['options'] and interpreter['options'].get( 'url', '').find('presto') >= 0: from notebook.connectors.jdbc_presto import JdbcApiPresto return JdbcApiPresto(request.user, interpreter=interpreter) elif interpreter['options'] and interpreter['options'].get( 'url', '').find('clickhouse') >= 0: from notebook.connectors.jdbc_clickhouse import JdbcApiClickhouse return JdbcApiClickhouse(request.user, interpreter=interpreter) else: from notebook.connectors.jdbc import JdbcApi return JdbcApi(request.user, interpreter=interpreter) elif interface == 'teradata': from notebook.connectors.jdbc import JdbcApiTeradata return JdbcApiTeradata(request.user, interpreter=interpreter) elif interface == 'athena': from notebook.connectors.jdbc import JdbcApiAthena return JdbcApiAthena(request.user, interpreter=interpreter) elif interface == 'presto': from notebook.connectors.jdbc_presto import JdbcApiPresto return JdbcApiPresto(request.user, interpreter=interpreter) elif interface == 'sqlalchemy': from notebook.connectors.sqlalchemyapi import SqlAlchemyApi return SqlAlchemyApi(request.user, interpreter=interpreter) elif interface == 'solr': from notebook.connectors.solr import SolrApi return SolrApi(request.user, interpreter=interpreter) elif interface == 'hbase': from notebook.connectors.hbase import HBaseApi return HBaseApi(request.user) elif interface == 'kafka': from notebook.connectors.kafka import KafkaApi return KafkaApi(request.user) elif interface == 'pig': return OozieApi(user=request.user, request=request) # Backward compatibility until Hue 4 else: raise PopupException( _('Notebook connector interface not recognized: %s') % interface)
def run_sqoop(request, source, destination, start_time): rdbms_mode = source['rdbmsMode'] rdbms_name = source['rdbmsJdbcDriverName'] if source[ 'rdbmsType'] == 'jdbc' else source['rdbmsType'] rdbms_database_name = source['rdbmsDatabaseName'] rdbms_all_tables_selected = source['rdbmsAllTablesSelected'] destination_type = destination['outputFormat'] destination_name = destination['name'] destination_table_name = destination['tableName'] destination_database_name = destination['databaseName'] destination_mappers_num = destination['numMappers'] destination_file_output_format = destination['rdbmsFileOutputFormat'] destination_custom_fields_delimiter = destination['customFieldsDelimiter'] destination_custom_line_delimiter = destination['customLineDelimiter'] destination_custom_enclosed_by_delimiter = destination[ 'customEnclosedByDelimiter'] destination_splitby_column = destination['rdbmsSplitByColumn'] if not rdbms_all_tables_selected: rdbms_table_name = source['rdbmsTableName'] if rdbms_mode == 'configRdbms' and rdbms_name == 'jdbc': username = '' password = '' url = '' interpreters = get_ordered_interpreters(request.user) key = [ key for key in interpreters if key['name'] == source['rdbmsJdbcDriverName'] ] if key: options = key[0]['options'] url = options['url'] password = options['password'] username = options['user'] statement = '--connect %(url)s --username %(username)s --password %(password)s' % { 'url': url, 'username': username, 'password': password } else: if rdbms_mode == 'configRdbms': rdbms_host = DATABASES[rdbms_name].HOST.get() rdbms_port = DATABASES[rdbms_name].PORT.get() rdbms_user_name = DATABASES[rdbms_name].USER.get() rdbms_password = get_database_password(rdbms_name) else: rdbms_host = source['rdbmsHostname'] rdbms_port = source['rdbmsPort'] rdbms_user_name = source['rdbmsUsername'] rdbms_password = source['rdbmsPassword'] password_file_path = request.fs.join( request.fs.get_home_dir() + '/sqoop/', uuid.uuid4().hex + '.password') request.fs.do_as_user(request.user, request.fs.create, password_file_path, overwrite=True, permission=0700, data=smart_str(rdbms_password)) lib_files = [] if destination['sqoopJobLibPaths']: lib_files = [{ 'path': f['path'], 'type': 'jar' } for f in destination['sqoopJobLibPaths']] statement = '--connect jdbc:%(rdbmsName)s://%(rdbmsHost)s:%(rdbmsPort)s/%(rdbmsDatabaseName)s --username %(rdbmsUserName)s --password-file %(passwordFilePath)s' % { 'rdbmsName': rdbms_name, 'rdbmsHost': rdbms_host, 'rdbmsPort': rdbms_port, 'rdbmsDatabaseName': rdbms_database_name, 'rdbmsUserName': rdbms_user_name, 'passwordFilePath': password_file_path } if destination_type == 'file': success_url = '/filebrowser/view/' + destination_name targetDir = request.fs.fs_defaultfs + destination_name if rdbms_all_tables_selected: statement = 'import-all-tables %(statement)s --warehouse-dir %(targetDir)s' % { 'statement': statement, 'targetDir': targetDir } else: statement = 'import %(statement)s --table %(rdbmsTableName)s --delete-target-dir --target-dir %(targetDir)s' % { 'statement': statement, 'rdbmsTableName': rdbms_table_name, 'targetDir': targetDir } if destination_file_output_format == 'text': statement = '%(statement)s --as-textfile --fields-terminated-by %(customFieldsDelimiter)s --lines-terminated-by %(customLineDelimiter)s --enclosed-by %(customEnclosedByDelimiter)s' % { 'statement': statement, 'customFieldsDelimiter': destination_custom_fields_delimiter, 'customLineDelimiter': destination_custom_line_delimiter, 'customEnclosedByDelimiter': destination_custom_enclosed_by_delimiter } elif destination_file_output_format == 'sequence': statement = '%(statement)s --as-sequencefile' % { 'statement': statement } elif destination_file_output_format == 'avro': statement = '%(statement)s --as-avrodatafile' % { 'statement': statement } statement = _splitby_column_check(statement, destination_splitby_column) elif destination_type == 'table': success_url = reverse('metastore:describe_table', kwargs={ 'database': destination_database_name, 'table': destination_table_name }) if rdbms_all_tables_selected: statement = 'import-all-tables %(statement)s --hive-import' % { 'statement': statement } else: statement = 'import %(statement)s --table %(rdbmsTableName)s --hive-import' % { 'statement': statement, 'rdbmsTableName': rdbms_table_name } statement = _splitby_column_check(statement, destination_splitby_column) elif destination_type == 'hbase': success_url = '/hbase/#HBase/' + destination_table_name # Todo statement = '%(statement)s --num-mappers %(numMappers)s' % { 'statement': statement, 'numMappers': destination_mappers_num } task = make_notebook(name=_( 'Indexer job for %(rdbmsDatabaseName)s.%(rdbmsDatabaseName)s to %(path)s' ) % { 'rdbmsDatabaseName': rdbms_database_name, 'rdbmsDatabaseName': rdbms_database_name, 'path': destination_name }, editor_type='sqoop1', statement=statement, files=lib_files, status='ready', on_success_url=success_url, last_executed=start_time, is_task=True) return task.execute(request, batch=False)