示例#1
0
文件: dbms.py 项目: 9629831527/hue
def get_query_server_config(name='beeswax', server=None):
  if name == 'impala':
    from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
        IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S

    query_server = {
        'server_name': 'impala',
        'server_host': IMPALA_SERVER_HOST.get(),
        'server_port': IMPALA_SERVER_PORT.get(),
        'principal': IMPALA_PRINCIPAL.get(),
        'impersonation_enabled': IMPERSONATION_ENABLED.get(),
        'querycache_rows': QUERYCACHE_ROWS.get(),
        'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
    }
  else:
    kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())

    query_server = {
        'server_name': 'beeswax', # Aka HiveServer2 now
        'server_host': HIVE_SERVER_HOST.get(),
        'server_port': HIVE_SERVER_PORT.get(),
        'principal': kerberos_principal
    }

  LOG.debug("Query Server: %s" % query_server)

  return query_server
示例#2
0
def get_query_server_config(name='beeswax', server=None):
  if name == 'impala':
    from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
        IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS

    query_server = {
        'server_name': 'impala',
        'server_host': IMPALA_SERVER_HOST.get(),
        'server_port': IMPALA_SERVER_PORT.get(),
        'principal': IMPALA_PRINCIPAL.get(),
        'impersonation_enabled': IMPERSONATION_ENABLED.get(),
        'querycache_rows': QUERYCACHE_ROWS.get()
    }
  else:
    kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())

    query_server = {
        'server_name': 'beeswax', # Aka HiveServer2 now
        'server_host': HIVE_SERVER_HOST.get(),
        'server_port': HIVE_SERVER_PORT.get(),
        'principal': kerberos_principal
    }

  LOG.debug("Query Server: %s" % query_server)

  return query_server
示例#3
0
def get_query_server_config(name='beeswax', server=None):
  if name == 'impala':
    from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
        IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S

    query_server = {
        'server_name': 'impala',
        'server_host': IMPALA_SERVER_HOST.get(),
        'server_port': IMPALA_SERVER_PORT.get(),
        'principal': IMPALA_PRINCIPAL.get(),
        'impersonation_enabled': IMPERSONATION_ENABLED.get(),
        'querycache_rows': QUERYCACHE_ROWS.get(),
        'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
    }
  else:
    kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())

    query_server = {
        'server_name': 'beeswax', # Aka HiveServer2 now
        'server_host': HIVE_SERVER_HOST.get(),
        'server_port': HIVE_SERVER_PORT.get(),
        'principal': kerberos_principal,
        'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
            'protocol': 'https' if hiveserver2_use_ssl() else 'http',
            'host': HIVE_SERVER_HOST.get(),
            'port': hive_site.hiveserver2_thrift_http_port(),
            'end_point': hive_site.hiveserver2_thrift_http_path()
        },
        'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
    }

  LOG.debug("Query Server: %s" % query_server)

  return query_server
示例#4
0
文件: dbms.py 项目: tempbottle/hue
def get_query_server_config(name='beeswax', server=None):
    if name == 'impala':
        from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
            IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S, AUTH_USERNAME as IMPALA_AUTH_USERNAME, AUTH_PASSWORD as IMPALA_AUTH_PASSWORD

        query_server = {
            'server_name': 'impala',
            'server_host': IMPALA_SERVER_HOST.get(),
            'server_port': IMPALA_SERVER_PORT.get(),
            'principal': IMPALA_PRINCIPAL.get(),
            'impersonation_enabled': IMPERSONATION_ENABLED.get(),
            'querycache_rows': QUERYCACHE_ROWS.get(),
            'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
            'auth_username': IMPALA_AUTH_USERNAME.get(),
            'auth_password': IMPALA_AUTH_PASSWORD.get()
        }
    else:
        kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
            HIVE_SERVER_HOST.get())

        query_server = {
            'server_name':
            'beeswax',  # Aka HiveServer2 now
            'server_host':
            HIVE_SERVER_HOST.get(),
            'server_port':
            HIVE_SERVER_PORT.get(),
            'principal':
            kerberos_principal,
            'http_url':
            '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
                'protocol': 'https' if hiveserver2_use_ssl() else 'http',
                'host': HIVE_SERVER_HOST.get(),
                'port': hive_site.hiveserver2_thrift_http_port(),
                'end_point': hive_site.hiveserver2_thrift_http_path()
            },
            'transport_mode':
            'http'
            if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
            'auth_username':
            AUTH_USERNAME.get(),
            'auth_password':
            AUTH_PASSWORD.get()
        }

    if name == 'sparksql':  # Spark SQL is almost the same as Hive
        from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT

        query_server.update({
            'server_name': 'sparksql',
            'server_host': SPARK_SERVER_HOST.get(),
            'server_port': SPARK_SERVER_PORT.get()
        })

    debug_query_server = query_server.copy()
    debug_query_server['auth_password_used'] = bool(
        debug_query_server.pop('auth_password'))
    LOG.debug("Query Server: %s" % debug_query_server)

    return query_server
示例#5
0
文件: dbms.py 项目: jounex/hue
def get_query_server_config(name='beeswax', server=None):
  if name == 'impala':
    from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
        IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S

    query_server = {
        'server_name': 'impala',
        'server_host': IMPALA_SERVER_HOST.get(),
        'server_port': IMPALA_SERVER_PORT.get(),
        'principal': IMPALA_PRINCIPAL.get(),
        'impersonation_enabled': IMPERSONATION_ENABLED.get(),
        'querycache_rows': QUERYCACHE_ROWS.get(),
        'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
    }
  else:
    kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())

    query_server = {
        'server_name': 'beeswax', # Aka HiveServer2 now
        'server_host': HIVE_SERVER_HOST.get(),
        'server_port': HIVE_SERVER_PORT.get(),
        'principal': kerberos_principal,
        'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
            'protocol': 'https' if hiveserver2_use_ssl() else 'http',
            'host': HIVE_SERVER_HOST.get(),
            'port': hive_site.hiveserver2_thrift_http_port(),
            'end_point': hive_site.hiveserver2_thrift_http_path()
        },
        'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
    }

  LOG.debug("Query Server: %s" % query_server)

  return query_server
示例#6
0
文件: tests.py 项目: iAmGhost/hue
def test_dump_config():
  c = make_logged_in_client()

  CANARY = "abracadabra"

  # Depending on the order of the conf.initialize() in settings, the set_for_testing() are not seen in the global settings variable
  clear = HIVE_SERVER_HOST.set_for_testing(CANARY)

  response1 = c.get(reverse('desktop.views.dump_config'))
  assert_true(CANARY in response1.content, response1.content)

  response2 = c.get(reverse('desktop.views.dump_config'), dict(private="true"))
  assert_true(CANARY in response2.content)

  # There are more private variables...
  assert_true(len(response1.content) < len(response2.content))

  clear()

  CANARY = "(localhost|127\.0\.0\.1):(50030|50070|50060|50075)"
  clear = proxy.conf.WHITELIST.set_for_testing(CANARY)

  response1 = c.get(reverse('desktop.views.dump_config'))
  assert_true(CANARY in response1.content)

  clear()

  # Malformed port per HUE-674
  CANARY = "asdfoijaoidfjaosdjffjfjaoojosjfiojdosjoidjfoa"
  clear = HIVE_SERVER_HOST.set_for_testing(CANARY)

  response1 = c.get(reverse('desktop.views.dump_config'))
  assert_true(CANARY in response1.content, response1.content)

  clear()

  CANARY = '/tmp/spacé.dat'
  finish = proxy.conf.WHITELIST.set_for_testing(CANARY)
  try:
    response = c.get(reverse('desktop.views.dump_config'))
    assert_true(CANARY in response.content, response.content)
  finally:
    finish()

  # Not showing some passwords
  response = c.get(reverse('desktop.views.dump_config'))
  assert_false('bind_password' in response.content)

  # Login as someone else
  client_not_me = make_logged_in_client(username='******', is_superuser=False, groupname='test')
  grant_access("not_me", "test", "desktop")

  response = client_not_me.get(reverse('desktop.views.dump_config'))
  assert_true("You must be a superuser" in response.content, response.content)

  os.environ["HUE_CONF_DIR"] = "/tmp/test_hue_conf_dir"
  resp = c.get(reverse('desktop.views.dump_config'))
  del os.environ["HUE_CONF_DIR"]
  assert_true('/tmp/test_hue_conf_dir' in resp.content, resp)
示例#7
0
def test_dump_config():
  c = make_logged_in_client()

  CANARY = "abracadabra"

  # Depending on the order of the conf.initialize() in settings, the set_for_testing() are not seen in the global settings variable
  clear = HIVE_SERVER_HOST.set_for_testing(CANARY)

  response1 = c.get(reverse('desktop.views.dump_config'))
  assert_true(CANARY in response1.content, response1.content)

  response2 = c.get(reverse('desktop.views.dump_config'), dict(private="true"))
  assert_true(CANARY in response2.content)

  # There are more private variables...
  assert_true(len(response1.content) < len(response2.content))

  clear()

  CANARY = "(localhost|127\.0\.0\.1):(50030|50070|50060|50075)"
  clear = proxy.conf.WHITELIST.set_for_testing(CANARY)

  response1 = c.get(reverse('desktop.views.dump_config'))
  assert_true(CANARY in response1.content)

  clear()

  # Malformed port per HUE-674
  CANARY = "asdfoijaoidfjaosdjffjfjaoojosjfiojdosjoidjfoa"
  clear = HIVE_SERVER_HOST.set_for_testing(CANARY)

  response1 = c.get(reverse('desktop.views.dump_config'))
  assert_true(CANARY in response1.content, response1.content)

  clear()

  CANARY = '/tmp/spacé.dat'
  finish = proxy.conf.WHITELIST.set_for_testing(CANARY)
  try:
    response = c.get(reverse('desktop.views.dump_config'))
    assert_true(CANARY in response.content, response.content)
  finally:
    finish()

  # Not showing some passwords
  response = c.get(reverse('desktop.views.dump_config'))
  assert_false('bind_password' in response.content)

  # Login as someone else
  client_not_me = make_logged_in_client(username='******', is_superuser=False, groupname='test')
  grant_access("not_me", "test", "desktop")

  response = client_not_me.get(reverse('desktop.views.dump_config'))
  assert_true("You must be a superuser" in response.content, response.content)

  os.environ["HUE_CONF_DIR"] = "/tmp/test_hue_conf_dir"
  resp = c.get(reverse('desktop.views.dump_config'))
  del os.environ["HUE_CONF_DIR"]
  assert_true('/tmp/test_hue_conf_dir' in resp.content, resp)
示例#8
0
文件: dbms.py 项目: RoxC/hue
def get_query_server_config(name="beeswax", server=None):
    if name == "impala":
        from impala.conf import (
            SERVER_HOST as IMPALA_SERVER_HOST,
            SERVER_PORT as IMPALA_SERVER_PORT,
            IMPALA_PRINCIPAL,
            IMPERSONATION_ENABLED,
            QUERYCACHE_ROWS,
            QUERY_TIMEOUT_S,
            AUTH_USERNAME as IMPALA_AUTH_USERNAME,
            AUTH_PASSWORD as IMPALA_AUTH_PASSWORD,
        )

        query_server = {
            "server_name": "impala",
            "server_host": IMPALA_SERVER_HOST.get(),
            "server_port": IMPALA_SERVER_PORT.get(),
            "principal": IMPALA_PRINCIPAL.get(),
            "impersonation_enabled": IMPERSONATION_ENABLED.get(),
            "querycache_rows": QUERYCACHE_ROWS.get(),
            "QUERY_TIMEOUT_S": QUERY_TIMEOUT_S.get(),
            "auth_username": IMPALA_AUTH_USERNAME.get(),
            "auth_password": IMPALA_AUTH_PASSWORD.get(),
        }
    else:
        kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())

        query_server = {
            "server_name": "beeswax",  # Aka HiveServer2 now
            "server_host": HIVE_SERVER_HOST.get(),
            "server_port": HIVE_SERVER_PORT.get(),
            "principal": kerberos_principal,
            "http_url": "%(protocol)s://%(host)s:%(port)s/%(end_point)s"
            % {
                "protocol": "https" if hiveserver2_use_ssl() else "http",
                "host": HIVE_SERVER_HOST.get(),
                "port": hive_site.hiveserver2_thrift_http_port(),
                "end_point": hive_site.hiveserver2_thrift_http_path(),
            },
            "transport_mode": "http" if hive_site.hiveserver2_transport_mode() == "HTTP" else "socket",
            "auth_username": AUTH_USERNAME.get(),
            "auth_password": AUTH_PASSWORD.get(),
        }

    if name == "sparksql":  # Spark SQL is almost the same as Hive
        from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT

        query_server.update(
            {"server_name": "sparksql", "server_host": SPARK_SERVER_HOST.get(), "server_port": SPARK_SERVER_PORT.get()}
        )

    debug_query_server = query_server.copy()
    debug_query_server["auth_password_used"] = bool(debug_query_server.pop("auth_password"))
    LOG.debug("Query Server: %s" % debug_query_server)

    return query_server
示例#9
0
def get_query_server_config(name='beeswax', server=None, cluster=None):
    if cluster and cluster != CLUSTER_ID.get():
        cluster_config = Cluster(user=None).get_config(cluster)
    else:
        cluster_config = None

    if name == 'impala':
        from impala.dbms import get_query_server_config as impala_query_server_config
        query_server = impala_query_server_config(
            cluster_config=cluster_config)
    else:
        kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
            HIVE_SERVER_HOST.get())

        query_server = {
            'server_name':
            name if not cluster_config else '%s-%s' %
            (name, cluster_config.get('id')),
            'server_host':
            HIVE_SERVER_HOST.get()
            if not cluster_config else cluster_config.get('server_host'),
            'server_port':
            HIVE_SERVER_PORT.get(),
            'principal':
            kerberos_principal,
            'http_url':
            '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
                'protocol': 'https' if hiveserver2_use_ssl() else 'http',
                'host': HIVE_SERVER_HOST.get(),
                'port': hive_site.hiveserver2_thrift_http_port(),
                'end_point': hive_site.hiveserver2_thrift_http_path()
            },
            'transport_mode':
            'http'
            if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
            'auth_username':
            AUTH_USERNAME.get(),
            'auth_password':
            AUTH_PASSWORD.get()
        }

    if name == 'sparksql':  # Spark SQL is almost the same as Hive
        from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT

        query_server.update({
            'server_name': 'sparksql',
            'server_host': SPARK_SERVER_HOST.get(),
            'server_port': SPARK_SERVER_PORT.get()
        })

    debug_query_server = query_server.copy()
    debug_query_server['auth_password_used'] = bool(
        debug_query_server.pop('auth_password'))
    LOG.debug("Query Server: %s" % debug_query_server)

    return query_server
示例#10
0
文件: dbms.py 项目: rhmiller47/hue
def get_query_server_config(name='beeswax', server=None):
  if name == 'impala':
    from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
        IMPALA_PRINCIPAL, IMPERSONATION_ENABLED, QUERYCACHE_ROWS, QUERY_TIMEOUT_S, AUTH_USERNAME as IMPALA_AUTH_USERNAME, AUTH_PASSWORD as IMPALA_AUTH_PASSWORD, \
        SESSION_TIMEOUT_S

    query_server = {
        'server_name': 'impala',
        'server_host': IMPALA_SERVER_HOST.get(),
        'server_port': IMPALA_SERVER_PORT.get(),
        'principal': IMPALA_PRINCIPAL.get(),
        'impersonation_enabled': IMPERSONATION_ENABLED.get(),
        'querycache_rows': QUERYCACHE_ROWS.get(),
        'QUERY_TIMEOUT_S': QUERY_TIMEOUT_S.get(),
        'SESSION_TIMEOUT_S': SESSION_TIMEOUT_S.get(),
        'auth_username': IMPALA_AUTH_USERNAME.get(),
        'auth_password': IMPALA_AUTH_PASSWORD.get()
    }
  else:
    kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())

    query_server = {
        'server_name': 'beeswax', # Aka HiveServer2 now
        'server_host': HIVE_SERVER_HOST.get(),
        'server_port': HIVE_SERVER_PORT.get(),
        'principal': kerberos_principal,
        'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
            'protocol': 'https' if hiveserver2_use_ssl() else 'http',
            'host': HIVE_SERVER_HOST.get(),
            'port': hive_site.hiveserver2_thrift_http_port(),
            'end_point': hive_site.hiveserver2_thrift_http_path()
        },
        'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
        'auth_username': AUTH_USERNAME.get(),
        'auth_password': AUTH_PASSWORD.get()
    }

  if name == 'sparksql': # Spark SQL is almost the same as Hive
    from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT

    query_server.update({
        'server_name': 'sparksql',
        'server_host': SPARK_SERVER_HOST.get(),
        'server_port': SPARK_SERVER_PORT.get()
    })

  debug_query_server = query_server.copy()
  debug_query_server['auth_password_used'] = bool(debug_query_server.pop('auth_password'))
  LOG.debug("Query Server: %s" % debug_query_server)

  return query_server
示例#11
0
文件: dbms.py 项目: cloudera/hue
def get_query_server_config(name='beeswax', server=None, cluster=None):
  LOG.debug("Query cluster %s: %s" % (name, cluster))

  cluster_config = get_cluster_config(cluster)

  if name == 'impala':
    from impala.dbms import get_query_server_config as impala_query_server_config
    query_server = impala_query_server_config(cluster_config=cluster_config)
  elif name == 'hms':
    kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
    query_server = {
        'server_name': 'hms',
        'server_host': HIVE_METASTORE_HOST.get() if not cluster_config else cluster_config.get('server_host'),
        'server_port': HIVE_METASTORE_PORT.get(),
        'principal': kerberos_principal,
        'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
        'auth_username': AUTH_USERNAME.get(),
        'auth_password': AUTH_PASSWORD.get()
    }
  else:
    kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
    query_server = {
        'server_name': 'beeswax',
        'server_host': HIVE_SERVER_HOST.get() if not cluster_config else cluster_config.get('server_host'),
        'server_port': HIVE_SERVER_PORT.get(),
        'principal': kerberos_principal,
        'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
            'protocol': 'https' if hiveserver2_use_ssl() else 'http',
            'host': HIVE_SERVER_HOST.get(),
            'port': hive_site.hiveserver2_thrift_http_port(),
            'end_point': hive_site.hiveserver2_thrift_http_path()
        },
        'transport_mode': 'http' if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
        'auth_username': AUTH_USERNAME.get(),
        'auth_password': AUTH_PASSWORD.get()
    }

  if name == 'sparksql': # Spark SQL is almost the same as Hive
    from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT

    query_server.update({
        'server_name': 'sparksql',
        'server_host': SPARK_SERVER_HOST.get(),
        'server_port': SPARK_SERVER_PORT.get()
    })

  debug_query_server = query_server.copy()
  debug_query_server['auth_password_used'] = bool(debug_query_server.pop('auth_password'))
  LOG.debug("Query Server: %s" % debug_query_server)

  return query_server
示例#12
0
    def test_get_hue_config(self):
        client = make_logged_in_client(username="******",
                                       groupname="default",
                                       recreate=True,
                                       is_superuser=True)
        user = User.objects.get(username="******")
        grant_access(user.username, user.username, "desktop")
        response = client.get('/desktop/api2/get_hue_config', data={})

        # It should have multiple config sections in json
        config = json.loads(response.content)['config']
        assert_true(len(config) > 1)

        # It should only allow superusers
        client_not_me = make_logged_in_client(username='******',
                                              is_superuser=False,
                                              groupname='test')
        grant_access("not_me", "test", "desktop")
        response = client_not_me.get('/desktop/api2/get_hue_config', data={})
        assert_true(b"You must be a superuser" in response.content,
                    response.content)

        # It should contain a config parameter
        CANARY = b"abracadabra"
        clear = HIVE_SERVER_HOST.set_for_testing(CANARY)
        try:
            response = client.get('/desktop/api2/get_hue_config', data={})
            assert_true(CANARY in response.content, response.content)
        finally:
            clear()
示例#13
0
文件: dbms.py 项目: jackesh/hue
def get_query_server_config(name='beeswax', server=None):
  if name == 'impala':
    from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, \
        IMPALA_PRINCIPAL, IMPERSONATION_ENABLED

    query_server = {
        'server_name': 'impala',
        'server_host': IMPALA_SERVER_HOST.get(),
        'server_port': IMPALA_SERVER_PORT.get(),
        'principal': IMPALA_PRINCIPAL.get(),
        'impersonation_enabled': IMPERSONATION_ENABLED.get()
    }
  elif name == 'rdbms':
    from rdbms.conf import RDBMS

    if not server or server not in RDBMS:
      keys = RDBMS.keys()
      name = keys and keys[0] or None
    else:
      name = server

    if name:
      query_server = {
        'server_name': RDBMS[name].ENGINE.get().split('.')[-1],
        'server_host': RDBMS[name].HOST.get(),
        'server_port': RDBMS[name].PORT.get(),
        'username': RDBMS[name].USER.get(),
        'password': RDBMS[name].PASSWORD.get(),
        'password': RDBMS[name].PASSWORD.get(),
        'alias': name
      }
    else:
      query_server = {}

  else:
    kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())

    query_server = {
        'server_name': 'beeswax', # Aka HiveServer2 now
        'server_host': HIVE_SERVER_HOST.get(),
        'server_port': HIVE_SERVER_PORT.get(),
        'principal': kerberos_principal
    }

  LOG.debug("Query Server: %s" % query_server)

  return query_server
示例#14
0
def get_query_server_config(name='beeswax', server=None):
  kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())

  query_server = {
    'server_name': 'beeswax', # Aka HiveServer2 now
    'server_host': HIVE_SERVER_HOST.get(),
    'server_port': HIVE_SERVER_PORT.get(),
    'principal': kerberos_principal,
    'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
        'protocol': 'https' if SSL.ENABLED.get() else 'http',
        'host': HIVE_SERVER_HOST.get(),
        'port': hive_site.hiveserver2_thrift_http_port(),
        'end_point': hive_site.hiveserver2_thrift_http_path()
    },
    'transport_mode': hive_site.hiveserver2_transport_mode(),
  }

  LOG.debug("Query Server: %s" % query_server)

  return query_server
示例#15
0
文件: dbms.py 项目: cyc821211/hue
def get_query_server_config(name="beeswax", server=None):
    if name == "impala":
        from impala.conf import (
            SERVER_HOST as IMPALA_SERVER_HOST,
            SERVER_PORT as IMPALA_SERVER_PORT,
            IMPALA_PRINCIPAL,
            IMPERSONATION_ENABLED,
            QUERYCACHE_ROWS,
            QUERY_TIMEOUT_S,
        )

        query_server = {
            "server_name": "impala",
            "server_host": IMPALA_SERVER_HOST.get(),
            "server_port": IMPALA_SERVER_PORT.get(),
            "principal": IMPALA_PRINCIPAL.get(),
            "impersonation_enabled": IMPERSONATION_ENABLED.get(),
            "querycache_rows": QUERYCACHE_ROWS.get(),
            "QUERY_TIMEOUT_S": QUERY_TIMEOUT_S.get(),
        }
    else:
        kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())

        query_server = {
            "server_name": "beeswax",  # Aka HiveServer2 now
            "server_host": HIVE_SERVER_HOST.get(),
            "server_port": HIVE_SERVER_PORT.get(),
            "principal": kerberos_principal,
            "http_url": "%(protocol)s://%(host)s:%(port)s/%(end_point)s"
            % {
                "protocol": "https" if hiveserver2_use_ssl() else "http",
                "host": HIVE_SERVER_HOST.get(),
                "port": hive_site.hiveserver2_thrift_http_port(),
                "end_point": hive_site.hiveserver2_thrift_http_path(),
            },
            "transport_mode": "http" if hive_site.hiveserver2_transport_mode() == "HTTP" else "socket",
        }

    LOG.debug("Query Server: %s" % query_server)

    return query_server
示例#16
0
def get_query_server_config(name='beeswax', server=None):
    kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
        HIVE_SERVER_HOST.get())

    query_server = {
        'server_name': 'beeswax',  # Aka HiveServer2 now
        'server_host': HIVE_SERVER_HOST.get(),
        'server_port': HIVE_SERVER_PORT.get(),
        'principal': kerberos_principal,
        'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
            'protocol': 'https' if SSL.ENABLED.get() else 'http',
            'host': HIVE_SERVER_HOST.get(),
            'port': hive_site.hiveserver2_thrift_http_port(),
            'end_point': hive_site.hiveserver2_thrift_http_path()
        },
        'transport_mode': hive_site.hiveserver2_transport_mode(),
    }

    LOG.debug("Query Server: %s" % query_server)

    return query_server
示例#17
0
文件: dbms.py 项目: altoplano/hue
def get_query_server_config(name='beeswax'):
  if name == 'impala':
    from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, IMPALA_PRINCIPAL

    query_server = {
        'server_name': 'impala',
        'server_host': IMPALA_SERVER_HOST.get(),
        'server_port': IMPALA_SERVER_PORT.get(),
        'principal': IMPALA_PRINCIPAL.get(),
    }
  else:
    kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())

    query_server = {
        'server_name': 'beeswax', # Aka HiveServer2 now
        'server_host': HIVE_SERVER_HOST.get(),
        'server_port': HIVE_SERVER_PORT.get(),
        'principal': kerberos_principal
    }
    LOG.debug("Query Server: %s" % query_server)

  return query_server
示例#18
0
文件: dbms.py 项目: sagarvarute/hue
def get_query_server_config(name='beeswax'):
    if name == 'impala':
        from impala.conf import SERVER_HOST as IMPALA_SERVER_HOST, SERVER_PORT as IMPALA_SERVER_PORT, IMPALA_PRINCIPAL

        query_server = {
            'server_name': 'impala',
            'server_host': IMPALA_SERVER_HOST.get(),
            'server_port': IMPALA_SERVER_PORT.get(),
            'principal': IMPALA_PRINCIPAL.get(),
        }
    else:
        kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
            HIVE_SERVER_HOST.get())

        query_server = {
            'server_name': 'beeswax',  # Aka HiveServer2 now
            'server_host': HIVE_SERVER_HOST.get(),
            'server_port': HIVE_SERVER_PORT.get(),
            'principal': kerberos_principal
        }
        LOG.debug("Query Server: %s" % query_server)

    return query_server
示例#19
0
文件: dbms.py 项目: fnerdwq/hue
def get_query_server_config(name="beeswax", server=None):
    if name == "impala":
        from impala.dbms import get_query_server_config as impala_query_server_config

        query_server = impala_query_server_config()
    else:
        kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())

        query_server = {
            "server_name": "beeswax",  # Aka HiveServer2 now
            "server_host": HIVE_SERVER_HOST.get(),
            "server_port": HIVE_SERVER_PORT.get(),
            "principal": kerberos_principal,
            "http_url": "%(protocol)s://%(host)s:%(port)s/%(end_point)s"
            % {
                "protocol": "https" if hiveserver2_use_ssl() else "http",
                "host": HIVE_SERVER_HOST.get(),
                "port": hive_site.hiveserver2_thrift_http_port(),
                "end_point": hive_site.hiveserver2_thrift_http_path(),
            },
            "transport_mode": "http" if hive_site.hiveserver2_transport_mode() == "HTTP" else "socket",
            "auth_username": AUTH_USERNAME.get(),
            "auth_password": AUTH_PASSWORD.get(),
        }

    if name == "sparksql":  # Spark SQL is almost the same as Hive
        from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT

        query_server.update(
            {"server_name": "sparksql", "server_host": SPARK_SERVER_HOST.get(), "server_port": SPARK_SERVER_PORT.get()}
        )

    debug_query_server = query_server.copy()
    debug_query_server["auth_password_used"] = bool(debug_query_server.pop("auth_password"))
    LOG.debug("Query Server: %s" % debug_query_server)

    return query_server
示例#20
0
def get_query_server_config(name='beeswax', connector=None):
  if connector and has_connectors(): # TODO: Give empty connector when no connector in use
    LOG.debug("Query via connector %s" % name)
    query_server = get_query_server_config_via_connector(connector)
  else:
    LOG.debug("Query via ini %s" % name)
    if name == "llap":
      activeEndpoint = cache.get('llap')
      if activeEndpoint is None:
        if HIVE_DISCOVERY_LLAP.get():
          LOG.debug("Checking zookeeper for discovering Hive LLAP server endpoint")
          zk = KazooClient(hosts=libzookeeper_conf.ENSEMBLE.get(), read_only=True)
          zk.start()
          if HIVE_DISCOVERY_LLAP_HA.get():
            znode = "{0}/instances".format(HIVE_DISCOVERY_LLAP_ZNODE.get())
            LOG.debug("Setting up Hive LLAP HA with the following node {0}".format(znode))
            if zk.exists(znode):
              hiveservers = zk.get_children(znode)
              if not hiveservers:
                raise PopupException(_('There is no running Hive LLAP server available'))
              LOG.info("Available Hive LLAP servers: {0}".format(hiveservers))
              for server in hiveservers:
                llap_servers = json.loads(zk.get("{0}/{1}".format(znode, server))[0])["internal"][0]
                if llap_servers["api"] == "activeEndpoint":
                  LOG.info("Selecting Hive LLAP server: {0}".format(llap_servers))
                  cache.set(
                    "llap",
                    json.dumps({
                        "host": llap_servers["addresses"][0]["host"],
                        "port": llap_servers["addresses"][0]["port"]
                      }),
                      CACHE_TIMEOUT.get()
                  )
            else:
              LOG.error("Hive LLAP endpoint not found, reverting to config values")
              cache.set("llap", json.dumps({"host": HIVE_SERVER_HOST.get(), "port": HIVE_HTTP_THRIFT_PORT.get()}), CACHE_TIMEOUT.get())
          else:
            znode = "{0}".format(HIVE_DISCOVERY_LLAP_ZNODE.get())
            LOG.debug("Setting up Hive LLAP with the following node {0}".format(znode))
            if zk.exists(znode):
              hiveservers = zk.get_children(znode)
              for server in hiveservers:
                cache.set(
                  "llap",
                  json.dumps({
                    "host": server.split(';')[0].split('=')[1].split(":")[0],
                    "port": server.split(';')[0].split('=')[1].split(":")[1]
                  })
                )
          zk.stop()
        else:
          LOG.debug("Zookeeper discovery not enabled, reverting to config values")
          cache.set("llap", json.dumps({"host": LLAP_SERVER_HOST.get(), "port": LLAP_SERVER_THRIFT_PORT.get()}), CACHE_TIMEOUT.get())

      activeEndpoint = json.loads(cache.get("llap"))

    elif name != 'hms' and name != 'impala':
      activeEndpoint = cache.get("hiveserver2")
      if activeEndpoint is None:
        if HIVE_DISCOVERY_HS2.get():
          hiveservers = get_zk_hs2()
          LOG.debug("Available Hive Servers: {0}".format(hiveservers))
          if not hiveservers:
            raise PopupException(_('There is no running Hive server available'))
          server_to_use = 0
          LOG.debug("Selected Hive server {0}: {1}".format(server_to_use, hiveservers[server_to_use]))
          cache.set(
            "hiveserver2",
            json.dumps({
              "host": hiveservers[server_to_use].split(";")[0].split("=")[1].split(":")[0],
              "port": hiveservers[server_to_use].split(";")[0].split("=")[1].split(":")[1]
            })
          )
        else:
          cache.set("hiveserver2", json.dumps({"host": HIVE_SERVER_HOST.get(), "port": HIVE_HTTP_THRIFT_PORT.get()}))
      else:
        # Setting hs2 cache in-case there is no HS2 discovery
        cache.set("hiveserver2", json.dumps({"host": HIVE_SERVER_HOST.get(), "port": HIVE_HTTP_THRIFT_PORT.get()}))
        if HIVE_DISCOVERY_HS2.get():
          # Replace ActiveEndpoint if the current HS2 is down
          hiveservers = get_zk_hs2()
          if hiveservers is not None:
            server_to_use = 0
            hs2_host_name = hiveservers[server_to_use].split(";")[0].split("=")[1].split(":")[0]
            hs2_in_active_endpoint = hs2_host_name in activeEndpoint
            LOG.debug("Is the current HS2 active {0}".format(hs2_in_active_endpoint))
            if not hs2_in_active_endpoint:
              LOG.error(
                'Current HiveServer is down, working to connect with the next available HiveServer from Zookeeper')
              reset_ha()
              server_to_use = 0
              LOG.debug("Selected HiveServer {0}: {1}".format(server_to_use, hiveservers[server_to_use]))
              cache.set(
                "hiveserver2",
                json.dumps({
                  "host": hiveservers[server_to_use].split(";")[0].split("=")[1].split(":")[0],
                  "port": hiveservers[server_to_use].split(";")[0].split("=")[1].split(":")[1]
                })
              )

      activeEndpoint = json.loads(cache.get("hiveserver2"))

    if name == 'impala':
      from impala.dbms import get_query_server_config as impala_query_server_config
      query_server = impala_query_server_config()
    elif name == 'hms':
      kerberos_principal = get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
      query_server = {
          'server_name': 'hms',
          'server_host': HIVE_METASTORE_HOST.get() if not cluster_config else cluster_config.get('server_host'),
          'server_port': HIVE_METASTORE_PORT.get(),
          'principal': kerberos_principal,
          'transport_mode': 'http' if hiveserver2_transport_mode() == 'HTTP' else 'socket',
          'auth_username': AUTH_USERNAME.get(),
          'auth_password': AUTH_PASSWORD.get(),
          'use_sasl': HIVE_USE_SASL.get()
      }
    else:
      kerberos_principal = get_hiveserver2_kerberos_principal(HIVE_SERVER_HOST.get())
      query_server = {
          'server_name': 'beeswax' if name != 'hplsql' else 'hplsql',
          'server_host': activeEndpoint["host"],
          'server_port': LLAP_SERVER_PORT.get() if name == 'llap' else HIVE_SERVER_PORT.get(),
          'principal': kerberos_principal,
          'http_url': '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
              'protocol': 'https' if hiveserver2_use_ssl() else 'http',
              'host': activeEndpoint["host"],
              'port': activeEndpoint["port"],
              'end_point': hiveserver2_thrift_http_path()
            },
          'transport_mode': 'http' if hiveserver2_transport_mode() == 'HTTP' else 'socket',
          'auth_username': AUTH_USERNAME.get(),
          'auth_password': AUTH_PASSWORD.get(),
          'use_sasl': HIVE_USE_SASL.get(),
          'close_sessions': CLOSE_SESSIONS.get(),
          'has_session_pool': has_session_pool(),
          'max_number_of_sessions': MAX_NUMBER_OF_SESSIONS.get()
        }

    if name == 'sparksql':  # Extends Hive as very similar
      from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT, USE_SASL as SPARK_USE_SASL

      query_server.update({
          'server_name': 'sparksql',
          'server_host': SPARK_SERVER_HOST.get(),
          'server_port': SPARK_SERVER_PORT.get(),
          'use_sasl': SPARK_USE_SASL.get()
      })

  if not query_server.get('dialect'):
    query_server['dialect'] = query_server['server_name']

  debug_query_server = query_server.copy()
  debug_query_server['auth_password_used'] = bool(debug_query_server.pop('auth_password', None))
  LOG.debug("Query Server: %s" % debug_query_server)

  return query_server
示例#21
0
def get_query_server_config(name='beeswax', server=None, cluster=None):
    LOG.debug("Query cluster %s: %s" % (name, cluster))

    cluster_config = get_cluster_config(cluster)
    if name == "llap":
        activeEndpoint = cache.get('llap')
        if activeEndpoint is None:
            if HIVE_DISCOVERY_LLAP.get():
                LOG.debug(
                    "Checking zookeeper for Hive Server Interactive endpoint")
                zk = KazooClient(hosts=libzookeeper_conf.ENSEMBLE.get(),
                                 read_only=True)
                zk.start()
                if HIVE_DISCOVERY_LLAP_HA.get():
                    znode = "{0}/instances".format(
                        HIVE_DISCOVERY_LLAP_ZNODE.get())
                    LOG.debug(
                        "Setting up LLAP with the following node {0}".format(
                            znode))
                    if zk.exists(znode):
                        hiveservers = zk.get_children(znode)
                        for server in hiveservers:
                            llap_servers = json.loads(
                                zk.get("{0}/{1}".format(
                                    znode, server))[0])["internal"][0]
                            if llap_servers["api"] == "activeEndpoint":
                                cache.set(
                                    "llap",
                                    json.dumps({
                                        "host":
                                        llap_servers["addresses"][0]["host"],
                                        "port":
                                        llap_servers["addresses"][0]["port"]
                                    }), CACHE_TIMEOUT.get())
                    else:
                        LOG.error(
                            "LLAP Endpoint not found, reverting to HiveServer2"
                        )
                        cache.set(
                            "llap",
                            json.dumps({
                                "host":
                                HIVE_SERVER_HOST.get(),
                                "port":
                                hive_site.hiveserver2_thrift_http_port()
                            }), CACHE_TIMEOUT.get())
                else:
                    znode = "{0}".format(HIVE_DISCOVERY_LLAP_ZNODE.get())
                    LOG.debug(
                        "Setting up LLAP with the following node {0}".format(
                            znode))
                    if zk.exists(znode):
                        hiveservers = zk.get_children(znode)
                        for server in hiveservers:
                            cache.set(
                                "llap",
                                json.dumps({
                                    "host":
                                    server.split(';')[0].split('=')[1].split(
                                        ":")[0],
                                    "port":
                                    server.split(';')[0].split('=')[1].split(
                                        ":")[1]
                                }))
                zk.stop()
            else:
                LOG.debug(
                    "Zookeeper Discovery not enabled, reverting to config values"
                )
                cache.set(
                    "llap",
                    json.dumps({
                        "host": LLAP_SERVER_HOST.get(),
                        "port": LLAP_SERVER_THRIFT_PORT.get()
                    }), CACHE_TIMEOUT.get())
        activeEndpoint = json.loads(cache.get("llap"))
    elif name != 'hms' and name != 'impala':
        activeEndpoint = cache.get("hiveserver2")
        if activeEndpoint is None:
            if HIVE_DISCOVERY_HS2.get():
                zk = KazooClient(hosts=libzookeeper_conf.ENSEMBLE.get(),
                                 read_only=True)
                zk.start()
                znode = HIVE_DISCOVERY_HIVESERVER2_ZNODE.get()
                LOG.info("Setting up Hive with the following node {0}".format(
                    znode))
                if zk.exists(znode):
                    hiveservers = zk.get_children(znode)
                    server_to_use = 0  # if CONF.HIVE_SPREAD.get() randint(0, len(hiveservers)-1) else 0
                    cache.set(
                        "hiveserver2",
                        json.dumps({
                            "host":
                            hiveservers[server_to_use].split(";")[0].split(
                                "=")[1].split(":")[0],
                            "port":
                            hiveservers[server_to_use].split(";")[0].split("=")
                            [1].split(":")[1]
                        }))
                else:
                    cache.set(
                        "hiveserver2",
                        json.dumps({
                            "host":
                            HIVE_SERVER_HOST.get(),
                            "port":
                            hive_site.hiveserver2_thrift_http_port()
                        }))
                zk.stop()
            else:
                cache.set(
                    "hiveserver2",
                    json.dumps({
                        "host": HIVE_SERVER_HOST.get(),
                        "port": hive_site.hiveserver2_thrift_http_port()
                    }))
        activeEndpoint = json.loads(cache.get("hiveserver2"))

    if name == 'impala':
        from impala.dbms import get_query_server_config as impala_query_server_config
        query_server = impala_query_server_config(
            cluster_config=cluster_config)
    elif name == 'hms':
        kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
            HIVE_SERVER_HOST.get())
        query_server = {
            'server_name':
            'hms',
            'server_host':
            HIVE_METASTORE_HOST.get()
            if not cluster_config else cluster_config.get('server_host'),
            'server_port':
            HIVE_METASTORE_PORT.get(),
            'principal':
            kerberos_principal,
            'transport_mode':
            'http'
            if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
            'auth_username':
            AUTH_USERNAME.get(),
            'auth_password':
            AUTH_PASSWORD.get()
        }
    else:
        kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
            HIVE_SERVER_HOST.get())
        query_server = {
            'server_name':
            'beeswax',
            'server_host':
            activeEndpoint["host"],
            'server_port':
            LLAP_SERVER_PORT.get()
            if name == 'llap' else HIVE_SERVER_PORT.get(),
            'principal':
            kerberos_principal,
            'http_url':
            '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
                'protocol': 'https' if hiveserver2_use_ssl() else 'http',
                'host': activeEndpoint["host"],
                'port': activeEndpoint["port"],
                'end_point': hive_site.hiveserver2_thrift_http_path()
            },
            'transport_mode':
            'http'
            if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
            'auth_username':
            AUTH_USERNAME.get(),
            'auth_password':
            AUTH_PASSWORD.get()
        }
    if name == 'sparksql':  # Spark SQL is almost the same as Hive
        from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT

        query_server.update({
            'server_name': 'sparksql',
            'server_host': SPARK_SERVER_HOST.get(),
            'server_port': SPARK_SERVER_PORT.get()
        })

    debug_query_server = query_server.copy()
    debug_query_server['auth_password_used'] = bool(
        debug_query_server.pop('auth_password'))
    LOG.debug("Query Server: %s" % debug_query_server)

    return query_server