Beispiel #1
0
def get_query_server_config(name='beeswax', server=None, cluster=None):
    if cluster and cluster != CLUSTER_ID.get():
        cluster_config = Cluster(user=None).get_config(cluster)
    else:
        cluster_config = None

    if name == 'impala':
        from impala.dbms import get_query_server_config as impala_query_server_config
        query_server = impala_query_server_config(
            cluster_config=cluster_config)
    else:
        kerberos_principal = hive_site.get_hiveserver2_kerberos_principal(
            HIVE_SERVER_HOST.get())

        query_server = {
            'server_name':
            name if not cluster_config else '%s-%s' %
            (name, cluster_config.get('id')),
            'server_host':
            HIVE_SERVER_HOST.get()
            if not cluster_config else cluster_config.get('server_host'),
            'server_port':
            HIVE_SERVER_PORT.get(),
            'principal':
            kerberos_principal,
            'http_url':
            '%(protocol)s://%(host)s:%(port)s/%(end_point)s' % {
                'protocol': 'https' if hiveserver2_use_ssl() else 'http',
                'host': HIVE_SERVER_HOST.get(),
                'port': hive_site.hiveserver2_thrift_http_port(),
                'end_point': hive_site.hiveserver2_thrift_http_path()
            },
            'transport_mode':
            'http'
            if hive_site.hiveserver2_transport_mode() == 'HTTP' else 'socket',
            'auth_username':
            AUTH_USERNAME.get(),
            'auth_password':
            AUTH_PASSWORD.get()
        }

    if name == 'sparksql':  # Spark SQL is almost the same as Hive
        from spark.conf import SQL_SERVER_HOST as SPARK_SERVER_HOST, SQL_SERVER_PORT as SPARK_SERVER_PORT

        query_server.update({
            'server_name': 'sparksql',
            'server_host': SPARK_SERVER_HOST.get(),
            'server_port': SPARK_SERVER_PORT.get()
        })

    debug_query_server = query_server.copy()
    debug_query_server['auth_password_used'] = bool(
        debug_query_server.pop('auth_password'))
    LOG.debug("Query Server: %s" % debug_query_server)

    return query_server
Beispiel #2
0
def get_cluster_config(cluster=None):
  if cluster and cluster.get('id') != CLUSTER_ID.get():
    if 'altus:dataware:k8s' in cluster['id']:
      compute_end_point = cluster['compute_end_point'][0] if type(cluster['compute_end_point']) == list else cluster['compute_end_point'] # TODO getting list from left assist
      cluster_config = {'server_host': compute_end_point, 'name': cluster['name']} # TODO get port too
    else:
      cluster_config = Cluster(user=None).get_config(cluster['id']) # Direct cluster
  else:
    cluster_config = None

  return cluster_config
Beispiel #3
0
def get_cluster_config(cluster=None):
  if cluster and cluster.get('connector'): # Connector interface
    cluster_config = cluster
  elif cluster and cluster.get('id') != CLUSTER_ID.get():
    if 'altus:dataware:k8s' in cluster['id']:
      compute_end_point = cluster['compute_end_point'][0] if type(cluster['compute_end_point']) == list else cluster['compute_end_point'] # TODO getting list from left assist
      cluster_config = {'server_host': compute_end_point, 'name': cluster['name']} # TODO get port too
    else:
      cluster_config = Cluster(user=None).get_config(cluster['id']) # Direct cluster # Deprecated
  else:
    cluster_config = None

  return cluster_config
Beispiel #4
0
    AUTH_KEY_SECRET_SCRIPT=Config(
      key="auth_key_secret_script",
      help=_t("Execute this script to produce the auth_key secret. This will be used when `auth_key_secret` is not set."),
      private=True,
      type=coerce_password_from_script,
      default=None),
    TENANT_ID=Config(
      key="tenant_id",
      help=_t("The name of the workload where queries are uploaded and optimizations are calculated from. Automatically guessed from auth_key and cluster_id if not specified."),
      private=True,
      default=None),
    CLUSTER_ID=Config(
      key="cluster_id",
      help=_t("The name of the cluster used to determine the tenant id when this one is not specified. Defaults to the cluster Id or 'default'."),
      private=True,
      default=DEFAULT_CLUSTER_ID.get()),

    APPLY_SENTRY_PERMISSIONS = Config(
      key="apply_sentry_permissions",
      help=_t("Perform Sentry privilege filtering. Default to true automatically if the cluster is secure."),
      dynamic_default=get_security_default,
      type=coerce_bool
    ),
    CACHEABLE_TTL=Config(
      key='cacheable_ttl',
      type=int,
      help=_t('The cache TTL in milliseconds for the assist/autocomplete/etc calls. Set to 0 to disable the cache.'),
      default=10 * 24 * 60 * 60 * 1000),
    AUTO_UPLOAD_QUERIES = Config(
      key="auto_upload_queries",
      help=_t("Automatically upload queries after their execution in order to improve recommendations."),
Beispiel #5
0
def _get_server_name(cluster_config):
    return 'impala' + ('-' + cluster_config.get('name') if cluster_config and
                       cluster_config.get('name') != CLUSTER_ID.get() else '')
Beispiel #6
0
def _get_server_name(cluster_config):
  return 'impala' + ('-' + cluster_config.get('name') if cluster_config and cluster_config.get('name') != CLUSTER_ID.get() else '')