def _query_store_proxy(request, path=None):
response = {'status': -1}
headers = {
'x-do-as': request.user.username,
'X-Requested-By': 'das',
'Content-Type': 'application/json; charset=UTF-8'
}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
if USE_SASL.get():
client.set_kerberos_auth()
try:
response = resource.invoke(request.method, path, request.GET.dict(),
request.body, headers)
except RestException as e:
ex_response = e.get_parent_ex().response
if ex_response is not None:
response['code'] = ex_response.status_code
response['message'] = ex_response.reason
response['content'] = ex_response.text
else:
response['message'] = 'Query store not reachable!'
response['content'] = e.message
return response
def flush(self):
data = self._wbuf.getvalue()
self._wbuf = buffer_writer()
# POST
self._root = Resource(self._client)
self._data = self._root.post('', data=data, headers=self._headers)
def query_store_api(request, path=None):
response = {'status': -1}
if USE_PROXY.get():
content_type = 'application/json; charset=UTF-8'
headers = {'X-Requested-By': 'das', 'Content-Type': content_type}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
if USE_SASL.get():
client.set_kerberos_auth()
try:
response = resource.invoke(request.method, path,
request.GET.dict(), request.body,
headers)
except RestException as e:
ex_response = e.get_parent_ex().response
response['code'] = ex_response.status_code
response['message'] = ex_response.reason
response['content'] = ex_response.text
else:
if path == 'api/query/search':
filters = json.loads(request.body)
resp = get_api(request.user,
interface='queries-hive').apps(filters['search'])
response = resp['apps']
return JsonResponse(response)
def get_task_log(self, offset=0):
logs = []
attempt = self.task.job.job_attempts['jobAttempt'][-1]
log_link = attempt['logsLink']
# Get MR task logs
if self.assignedContainerId:
log_link = log_link.replace(attempt['containerId'],
self.assignedContainerId)
if hasattr(self, 'nodeHttpAddress'):
log_link = log_link.replace(
attempt['nodeHttpAddress'].split(':')[0],
self.nodeHttpAddress.split(':')[0])
for name in ('stdout', 'stderr', 'syslog'):
link = '/%s/' % name
params = {}
if int(offset) >= 0:
params['start'] = offset
try:
log_link = re.sub('job_[^/]+', self.id, log_link)
root = Resource(get_log_client(log_link),
urlparse.urlsplit(log_link)[2],
urlencode=False)
response = root.get(link, params=params)
log = html.fromstring(response).xpath(
'/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s') % e
logs.append(log)
def __init__(self, user=None):
self._api_url = '%s/dw' % K8S.API_URL.get().rstrip('/')
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(False)
self._root = Resource(self._client)
def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = is_kerberos_enabled()
self._webserver_spnego_enabled = is_webserver_spnego_enabled()
self._thread_local = threading.local()
# You can set username/password for Impala Web UI which overrides kerberos
daemon_api_pwd = \
(DAEMON_API_PASSWORD.get() if DAEMON_API_PASSWORD.get()
is not None else (DAEMON_API_PASSWORD_SCRIPT.get() if DAEMON_API_PASSWORD_SCRIPT.get()
is not None else None))
if DAEMON_API_USERNAME.get(
) is not None and daemon_api_pwd is not None:
if DAEMON_API_AUTH_SCHEME.get().lower() == 'basic':
self._client.set_basic_auth(DAEMON_API_USERNAME.get(),
daemon_api_pwd)
LOG.info(
"Using username and password for basic authentication")
else:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(),
daemon_api_pwd)
LOG.info(
'Using username and password for digest authentication')
elif self._webserver_spnego_enabled or self._security_enabled:
self._client.set_kerberos_auth()
LOG.info('Using kerberos principal for authentication')
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = False
# To store user info
self._thread_local = threading.local()
def flush(self):
data = self._wbuf.getvalue()
self._wbuf = string_io()
# POST
self._root = Resource(self._client)
self._data = self._root.post('', data=data, headers=self._headers)
def get_kafka_topics(self, broker_host):
try:
client = HttpClient('http://%s:24042' % broker_host, logger=LOG)
root = Resource(client)
return root.get('/api/topics')
except RestException as e:
raise ManagerApiException(e)
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
def __init__(self, user=None, ssl_cert_ca_verify=False):
self._api_url = '%s/%s' % (PROMETHEUS.API_URL.get().strip('/'), VERSION)
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = Resource(self._client)
def __init__(self, solr_url, user):
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = SECURITY_ENABLED.get()
if self.security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
def __init__(self, user, api_url=None, auth_key=None, auth_key_secret=None, tenant_id='hue'):
self.user = user
self._tenant_id = tenant_id
self._api_url = (api_url or get_optimizer_url()).strip('/')
self._client = HttpClient(self._api_url, logger=LOG)
self._root = Resource(self._client)
self._api = MockApiLib()
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
self._thread_local = threading.local()
def __init__(self, livy_url):
self._url = posixpath.join(livy_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = SECURITY_ENABLED.get()
self._thread_local = threading.local()
if self.security_enabled:
self._client.set_kerberos_auth()
def __init__(self,
user=None,
security_enabled=False,
ssl_cert_ca_verify=False):
self._api_url = KAFKA.SCHEMA_REGISTRY_API_URL.get().strip(
'/') if KAFKA.SCHEMA_REGISTRY_API_URL.get() else ''
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._root = Resource(self._client)
def query_store_proxy(request, path=None):
content_type = 'application/json; charset=UTF-8'
headers = {'X-Requested-By': 'das', 'Content-Type': content_type}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
return JsonResponse(
resource.invoke(request.method, path, request.GET.dict(), request.body,
headers))
def __init__(self, username, mr_url, security_enabled=False, ssl_cert_ca_verify=False):
self._user = username
self._url = posixpath.join(mr_url, 'proxy')
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def flush(self):
if self.isOpen():
self.close()
self.open()
data = self._wbuf.getvalue()
self._wbuf = StringIO()
# POST
self._root = Resource(self._client)
self._data = self._root.post('', data=data)
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws/%s/history' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
self._thread_local = threading.local() # To store user info
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, spark_hs_url, security_enabled=False, ssl_cert_ca_verify=False):
self._ui_url = spark_hs_url
self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
# You can set username/password for Impala Web UI which overrides kerberos
if DAEMON_API_USERNAME.get() is not None and DAEMON_API_PASSWORD.get(
) is not None:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(),
DAEMON_API_PASSWORD.get())
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
def __init__(self, oozie_url, user, security_enabled=False, api_version=API_VERSION):
self._url = posixpath.join(oozie_url, api_version)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
if hasattr(user, 'username'):
self.user = user.username
else:
self.user = user
self.api_version = api_version
def query_store_download_bundle(request, id=None):
response = {}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
if USE_SASL.get():
client.set_kerberos_auth()
app = resource.get('api/data-bundle/' + id)
response = FileResponse((app, 'rb'), content_type='application/octet-stream')
response['Content-Disposition'] = 'attachment; filename=' + id + '.zip'
return response
def __init__(self,
oozie_url,
security_enabled=False,
ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
self._ssl_cert_ca_verify = ssl_cert_ca_verify
if self._security_enabled:
self._client.set_kerberos_auth()
if ssl_cert_ca_verify:
self._client.set_verify(True)
def __init__(self, livy_url):
self._url = posixpath.join(livy_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = SECURITY_ENABLED.get()
self._csrf_enabled = CSRF_ENABLED.get()
self._thread_local = threading.local()
if self.security_enabled:
self._client.set_kerberos_auth()
if self.csrf_enabled:
self._client.set_headers({'X-Requested-By': 'hue'})
self._client.set_verify(SSL_CERT_CA_VERIFY.get())
def app(self, appid):
query = self.api.get_query(query_id=appid)
if not query:
raise PopupException(_('Could not find query id %s' % appid))
params = {'extended': 'true', 'queryId': query.query_id}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
app = resource.get('api/hive/query',
params=params,
headers=self.HEADERS)
return app
def get_log_list(self):
log_link, user = self.get_log_link()
if not log_link:
return []
params = {'doAs': user}
log_link = re.sub('job_[^/]+', str(self.id), log_link)
root = Resource(get_log_client(log_link),
urlparse.urlsplit(log_link)[2],
urlencode=False)
response = root.get('/', params=params)
links = html.fromstring(response, parser=html.HTMLParser()).xpath(
'/html/body/table/tbody/tr/td[2]//a/@href')
parsed_links = map(lambda x: urlparse.urlsplit(x), links)
return map(lambda x: x and len(x) >= 2 and x[2].split('/')[-2] or '',
parsed_links)
def __init__(self,
user,
rm_url,
security_enabled=False,
ssl_cert_ca_verify=False):
self._user = user
self._url = posixpath.join(rm_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def test_resource_ascii():
with patch('desktop.lib.rest.http_client.HttpClient') as HttpClient:
client = HttpClient()
client.execute = Mock(
return_value=Mock(
headers={},
content='{"FileStatus":{"pathSuffix":"/user/hue/Джейкоб","type":"DIRECTORY","length":0,"owner":"admin","group":"admin","permission":"755","accessTime":0,"modificationTime":1578458822492,"blockSize":0,"replication":0,"childrenNum":0,"fileId":149137,"storagePolicy":0}}'
)
)
resource = Resource(client)
resource.get('/user/domain/Джейкоб')
assert_true(client._session)
