def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = is_kerberos_enabled()
self._webserver_spnego_enabled = is_webserver_spnego_enabled()
self._thread_local = threading.local()
# You can set username/password for Impala Web UI which overrides kerberos
daemon_api_pwd = \
(DAEMON_API_PASSWORD.get() if DAEMON_API_PASSWORD.get()
is not None else (DAEMON_API_PASSWORD_SCRIPT.get() if DAEMON_API_PASSWORD_SCRIPT.get()
is not None else None))
if DAEMON_API_USERNAME.get(
) is not None and daemon_api_pwd is not None:
if DAEMON_API_AUTH_SCHEME.get().lower() == 'basic':
self._client.set_basic_auth(DAEMON_API_USERNAME.get(),
daemon_api_pwd)
LOG.info(
"Using username and password for basic authentication")
else:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(),
daemon_api_pwd)
LOG.info(
'Using username and password for digest authentication')
elif self._webserver_spnego_enabled or self._security_enabled:
self._client.set_kerberos_auth()
LOG.info('Using kerberos principal for authentication')
def __init__(self, user=None):
self._api_url = '%s/dw' % K8S.API_URL.get().rstrip('/')
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(False)
self._root = Resource(self._client)
def __init__(self,
solr_url=None,
user=None,
security_enabled=False,
ssl_cert_ca_verify=SSL_CERT_CA_VERIFY.get()):
if solr_url is None:
solr_url = SOLR_URL.get()
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = security_enabled or SECURITY_ENABLED.get()
if self.security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
self._root = resource.Resource(self._client)
# The Kerberos handshake requires two requests in order to authenticate,
# but if our first request is a PUT/POST, it might flat-out reject the
# first request if the body is too large. So, connect here in order to get
# a cookie so future PUT/POSTs will be pre-authenticated.
if self.security_enabled:
self._root.invoke('HEAD', '/')
def _query_store_proxy(request, path=None):
response = {'status': -1}
headers = {
'x-do-as': request.user.username,
'X-Requested-By': 'das',
'Content-Type': 'application/json; charset=UTF-8'
}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
if USE_SASL.get():
client.set_kerberos_auth()
try:
response = resource.invoke(request.method, path, request.GET.dict(),
request.body, headers)
except RestException as e:
ex_response = e.get_parent_ex().response
if ex_response is not None:
response['code'] = ex_response.status_code
response['message'] = ex_response.reason
response['content'] = ex_response.text
else:
response['message'] = 'Query store not reachable!'
response['content'] = e.message
return response
def get_log_client(log_link):
global _log_client_queue
global MAX_HEAP_SIZE
_log_client_lock.acquire()
try:
components = urlparse.urlsplit(log_link)
base_url = '%(scheme)s://%(netloc)s' % {
'scheme': components[0],
'netloc': components[1]
}
# Takes on form (epoch time, client object)
# Least Recently Used algorithm.
client_tuple = next(
(tup for tup in _log_client_heap if tup[1].base_url == base_url),
None)
if client_tuple is None:
client = HttpClient(base_url, LOG)
yarn_cluster = cluster.get_cluster_conf_for_job_submission()
if yarn_cluster.SECURITY_ENABLED.get():
client.set_kerberos_auth()
else:
_log_client_heap.remove(client_tuple)
client = client_tuple[1]
new_client_tuple = (time.time(), client)
if len(_log_client_heap) >= MAX_HEAP_SIZE:
heapq.heapreplace(_log_client_heap, new_client_tuple)
else:
heapq.heappush(_log_client_heap, new_client_tuple)
return client
finally:
_log_client_lock.release()
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = False
# To store user info
self._thread_local = threading.local()
def query_store_api(request, path=None):
response = {'status': -1}
if USE_PROXY.get():
content_type = 'application/json; charset=UTF-8'
headers = {'X-Requested-By': 'das', 'Content-Type': content_type}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
if USE_SASL.get():
client.set_kerberos_auth()
try:
response = resource.invoke(request.method, path,
request.GET.dict(), request.body,
headers)
except RestException as e:
ex_response = e.get_parent_ex().response
response['code'] = ex_response.status_code
response['message'] = ex_response.reason
response['content'] = ex_response.text
else:
if path == 'api/query/search':
filters = json.loads(request.body)
resp = get_api(request.user,
interface='queries-hive').apps(filters['search'])
response = resp['apps']
return JsonResponse(response)
def get_access_token(cls, session_code, **options):
remote_url = options.get(
'remote_url', VCS[GITHUB_OFFICIAL].REMOTE_URL.get()).strip('/')
client_id = options.get('client_id',
VCS[GITHUB_OFFICIAL].CLIENT_ID.get())
client_secret = options.get('client_secret',
VCS[GITHUB_OFFICIAL].CLIENT_SECRET.get())
try:
client = HttpClient(remote_url, logger=LOG)
root = resource.Resource(client)
data = {
'client_id': client_id,
'client_secret': client_secret,
'code': session_code
}
headers = {
'content-type': 'application/json',
'Accept': 'application/json'
}
response = root.post('login/oauth/access_token',
headers=headers,
data=json.dumps(data))
result = cls._get_json(response)
return result['access_token']
except RestException as e:
raise GithubClientException(
'Failed to request access token from GitHub: %s' % e)
except KeyError:
raise GithubClientException(
'Failed to find access_token in GitHub oAuth response')
def __init__(self, base_url, cert_validate=True):
self._base_url = base_url
self._client = HttpClient(self._base_url,
logger=LOG,
cert_validate=cert_validate)
self._data = None
self._headers = None
self._wbuf = StringIO()
def __init__(self, solr_url, user):
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = SECURITY_ENABLED.get()
if self.security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
def __init__(self, user=None, ssl_cert_ca_verify=False):
self._api_url = '%s/%s' % (PROMETHEUS.API_URL.get().strip('/'), VERSION)
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = Resource(self._client)
def get_kafka_topics(self, broker_host):
try:
client = HttpClient('http://%s:24042' % broker_host, logger=LOG)
root = Resource(client)
return root.get('/api/topics')
except RestException as e:
raise ManagerApiException(e)
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
def __init__(self, user, api_url=None, auth_key=None, auth_key_secret=None, tenant_id='hue'):
self.user = user
self._tenant_id = tenant_id
self._api_url = (api_url or get_optimizer_url()).strip('/')
self._client = HttpClient(self._api_url, logger=LOG)
self._root = Resource(self._client)
self._api = MockApiLib()
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
self._thread_local = threading.local()
def __init__(self, livy_url):
self._url = posixpath.join(livy_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = SECURITY_ENABLED.get()
self._thread_local = threading.local()
if self.security_enabled:
self._client.set_kerberos_auth()
def query_store_proxy(request, path=None):
content_type = 'application/json; charset=UTF-8'
headers = {'X-Requested-By': 'das', 'Content-Type': content_type}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
return JsonResponse(
resource.invoke(request.method, path, request.GET.dict(), request.body,
headers))
def __init__(self, url, username, language='en'):
self._url = url
self._client = HttpClient(self._url, logger=LOG)
self._root = SqoopResource(self._client)
self._language = language
self._username = username
if has_sqoop_has_security():
self._client.set_kerberos_auth()
self._security_enabled = has_sqoop_has_security()
def __init__(self,
user=None,
security_enabled=False,
ssl_cert_ca_verify=False):
self._api_url = KAFKA.SCHEMA_REGISTRY_API_URL.get().strip(
'/') if KAFKA.SCHEMA_REGISTRY_API_URL.get() else ''
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._root = Resource(self._client)
def test_clear_cookies():
client = HttpClient('gethue')
client._session = MockedSession({'hue': 'rocks'})
client.execute('put', '/path')
assert_true(client._session.cookies)
client.execute('put', '/path', clear_cookies=True)
assert_false(client._session.cookies)
def __init__(self, api_url=None, user=None, password=None):
self._api_url = (api_url or NAVIGATOR.API_URL.get()).strip('/')
self._username = user or NAVIGATOR.AUTH_USERNAME.get()
self._password = password or NAVIGATOR.AUTH_PASSWORD.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_basic_auth(self._username, self._password)
self._root = resource.Resource(self._client)
self.__headers = {}
self.__params = ()
def __init__(self, spark_hs_url, security_enabled=False, ssl_cert_ca_verify=False):
self._ui_url = spark_hs_url
self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def is_authenticated(cls, access_token, **options):
api_url = options.get('api_url', GITHUB_API_URL.get()).strip('/')
try:
client = HttpClient(api_url, logger=LOG)
root = resource.Resource(client)
params = (('access_token', access_token), )
root.get('user', params=params)
return True
except RestException:
return False
def __init__(self, username, mr_url, security_enabled=False, ssl_cert_ca_verify=False):
self._user = username
self._url = posixpath.join(mr_url, 'proxy')
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws/%s/history' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
self._thread_local = threading.local() # To store user info
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None):
self._api_url = (api_url or OPTIMIZER.API_URL.get()).strip('/')
self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get()
self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get()
self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get()
self._email = OPTIMIZER.EMAIL.get()
self._email_password = OPTIMIZER.EMAIL_PASSWORD.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = resource.Resource(self._client)
def __init__(self, user=None):
self._api_url = '%s/%s' % (NAVIGATOR.API_URL.get().strip('/'), VERSION)
self._username = NAVIGATOR.AUTH_USERNAME.get()
self._password = NAVIGATOR.AUTH_PASSWORD.get()
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_basic_auth(self._username, self._password)
self._root = resource.Resource(self._client, urlencode=False) # For search_entities_interactive
self.__headers = {}
self.__params = ()
def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
# You can set username/password for Impala Web UI which overrides kerberos
if DAEMON_API_USERNAME.get() is not None and DAEMON_API_PASSWORD.get(
) is not None:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(),
DAEMON_API_PASSWORD.get())
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
def __init__(self, oozie_url, user, security_enabled=False, api_version=API_VERSION):
self._url = posixpath.join(oozie_url, api_version)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
if hasattr(user, 'username'):
self.user = user.username
else:
self.user = user
self.api_version = api_version
def __init__(self, **options):
self._github_base_url = options.get('remote_url', VCS[GITHUB_OFFICIAL].REMOTE_URL.get()).strip('/')
self._api_url = options.get('api_url', VCS[GITHUB_OFFICIAL].API_URL.get()).strip('/')
self._client = HttpClient(self._api_url, logger=LOG)
self._root = resource.Resource(self._client)
self.__headers = {}
access_token = options.get('access_token')
if access_token:
self.__headers['Authorization'] = 'token %s' % access_token
# TODO: Redact access_token from logs
self.__params = ()
