class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=True):
self._url = posixpath.join(oozie_url, "ws", _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "NodeManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def containers(self):
return self._root.get("node/containers", headers={"Accept": _JSON_CONTENT_TYPE})
def container(self, container_id):
return self._root.get(
"node/containers/%(container_id)s" % {"container_id": container_id}, headers={"Accept": _JSON_CONTENT_TYPE}
)
def query_store_api(request, path=None):
response = {'status': -1}
if USE_PROXY.get():
content_type = 'application/json; charset=UTF-8'
headers = {'X-Requested-By': 'das', 'Content-Type': content_type}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
if USE_SASL.get():
client.set_kerberos_auth()
try:
response = resource.invoke(request.method, path,
request.GET.dict(), request.body,
headers)
except RestException as e:
ex_response = e.get_parent_ex().response
response['code'] = ex_response.status_code
response['message'] = ex_response.reason
response['content'] = ex_response.text
else:
if path == 'api/query/search':
filters = json.loads(request.body)
resp = get_api(request.user,
interface='queries-hive').apps(filters['search'])
response = resp['apps']
return JsonResponse(response)
class ResourceManagerApi(object):
def __init__(self,
oozie_url,
security_enabled=False,
ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "ResourceManagerApi at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def cluster(self, **kwargs):
return self._execute(self._root.get,
'cluster',
params=kwargs,
headers={'Accept': _JSON_CONTENT_TYPE})
def apps(self, **kwargs):
return self._execute(self._root.get,
'cluster/apps',
params=kwargs,
headers={'Accept': _JSON_CONTENT_TYPE})
def app(self, app_id):
return self._execute(self._root.get,
'cluster/apps/%(app_id)s' % {'app_id': app_id},
headers={'Accept': _JSON_CONTENT_TYPE})
def kill(self, app_id):
return self._execute(self._root.put,
'cluster/apps/%(app_id)s/state' %
{'app_id': app_id},
data=json.dumps({'state': 'KILLED'}),
contenttype=_JSON_CONTENT_TYPE)
def _execute(self, function, *args, **kwargs):
response = function(*args, **kwargs)
# YARN-2605: Yarn does not use proper HTTP redirects when the standby RM has
# failed back to the master RM.
if isinstance(response, str) and response.startswith(
'This is standby RM. Redirecting to the current active RM'):
raise YarnFailoverOccurred(response)
return response
class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
self._ssl_cert_ca_verify = ssl_cert_ca_verify
if self._security_enabled:
self._client.set_kerberos_auth()
if ssl_cert_ca_verify:
self._client.set_verify(True)
def __str__(self):
return "ResourceManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def cluster(self, **kwargs):
return self._root.get('cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def apps(self, **kwargs):
return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def app(self, app_id):
return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def kill(self, app_id):
return self._root.put('cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE)
class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
def __str__(self):
return "ResourceManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def apps(self, **kwargs):
return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def app(self, app_id):
return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE})
class NodeManagerApi(object):
def __init__(self,
oozie_url,
security_enabled=False,
ssl_cert_ca_verify=True):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "NodeManagerApi at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def containers(self):
return self._root.get('node/containers',
headers={'Accept': _JSON_CONTENT_TYPE})
def container(self, container_id):
return self._root.get('node/containers/%(container_id)s' %
{'container_id': container_id},
headers={'Accept': _JSON_CONTENT_TYPE})
class NavigatorApi(object):
"""
http://cloudera.github.io/navigator/apidocs/v3/index.html
"""
def __init__(self, api_url=None, user=None, password=None):
self._api_url = '%s/%s' % ((api_url or NAVIGATOR.API_URL.get()).strip('/'), VERSION)
self._username = user or NAVIGATOR.AUTH_USERNAME.get()
self._password = password or NAVIGATOR.AUTH_PASSWORD.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_basic_auth(self._username, self._password)
self._root = resource.Resource(self._client)
self.__headers = {}
self.__params = ()
def search_entities(self, query_s, limit=100, offset=0, **filters):
"""
GET /api/v3/entities?query=()
http://cloudera.github.io/navigator/apidocs/v3/path__v3_entities.html
:param query_s: a query string of search terms (e.g. - sales quarterly);
Currently the search will perform an OR boolean search for all terms (split on whitespace), against a whitelist
of search_fields.
TODO: support smarter boolean searching with arbitrary ordering and precedence of conditionals
:param filters: TODO: IMPLEMENT ME, required to support property search
"""
search_fields = ('originalName', 'originalDescription', 'name', 'description', 'tags')
entity_types = ('DATABASE', 'TABLE', 'PARTITION', 'FIELD', 'FILE', 'OPERATION')
try:
params = self.__params
search_terms = [term.lower() for term in query_s.strip().split()]
query_clauses = []
for term in search_terms:
query_clauses.append('OR'.join(['(%s:*%s*)' % (field, term) for field in search_fields]))
filter_query = '(originalName:*.*)'
if search_terms:
filter_query = 'OR'.join(['(%s)' % clause for clause in query_clauses])
type_filter_clause = 'OR'.join(['(%s:%s)' % ('type', entity_type) for entity_type in entity_types])
filter_query = '%sAND(%s)' % (filter_query, type_filter_clause)
params += (
('query', filter_query),
('offset', offset),
('limit', limit),
)
response = self._root.get('entities', headers=self.__headers, params=params)
return response
except RestException, e:
msg = 'Failed to search for entities with search query: %s' % query_s
LOG.exception(msg)
raise NavigatorApiException(msg)
class OptimizerApi(object):
def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None):
self._api_url = (api_url or OPTIMIZER.API_URL.get()).strip('/')
self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get()
self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get()
self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get()
self._email = OPTIMIZER.EMAIL.get()
self._email_password = OPTIMIZER.EMAIL_PASSWORD.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = resource.Resource(self._client)
def create_product(self, product_name=None, product_secret=None, authCode=None):
try:
data = {
'productName': product_name if product_name is not None else self._product_name,
'productSecret': product_secret if product_secret is not None else self._product_secret,
'authCode': authCode if authCode is not None else self._product_auth_secret
}
return self._root.post('/api/createProduct', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
except RestException, e:
raise PopupException(e, title=_('Error while accessing Optimizer'))
class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
def __str__(self):
return "NodeManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def containers(self):
return self._root.get('node/containers', headers={'Accept': _JSON_CONTENT_TYPE})
def container(self, container_id):
return self._root.get('node/containers/%(container_id)s' % {'container_id': container_id}, headers={'Accept': _JSON_CONTENT_TYPE})
class SolrApi(object):
"""
http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
"""
def __init__(self, solr_url):
self._url = solr_url
self._client = HttpClient(self._url, logger=LOG)
if SECURITY_ENABLED.get():
self._client.set_kerberos_auth()
self._root = Resource(self._client)
def query(self, solr_query, hue_core):
try:
params = (
('q', solr_query['q'] or EMPTY_QUERY.get()),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
params += hue_core.get_query(solr_query)
fqs = solr_query['fq'].split('|')
for fq in fqs:
if fq:
params += (('fq', fq),)
response = self._root.get('%(collection)s/select' % solr_query, params)
if type(response) != dict:
# Got 'plain/text' mimetype instead of 'application/json'
response = json.loads(response)
return response
except RestException, e:
raise PopupException('Error while accessing Solr: %s' % e)
def __init__(self, user=None):
self._api_url = '%s/dw' % K8S.API_URL.get().rstrip('/')
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(False)
self._root = Resource(self._client)
class SolrApi(object):
"""
http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
"""
def __init__(self, solr_url, user):
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = SECURITY_ENABLED.get()
if self.security_enabled:
self._client.set_kerberos_auth()
self._root = resource.Resource(self._client)
def _get_params(self):
if self.security_enabled:
return (('doAs', self._user ),)
return (('user.name', DEFAULT_USER), ('doAs', self._user),)
@classmethod
def _get_json(cls, response):
if type(response) != dict:
# Got 'plain/text' mimetype instead of 'application/json'
try:
response = json.loads(response)
except ValueError, e:
# Got some null bytes in the response
LOG.error('%s: %s' % (unicode(e), repr(response)))
response = json.loads(response.replace('\x00', ''))
return response
def get_log_client(log_link):
global _log_client_queue
global MAX_HEAP_SIZE
_log_client_lock.acquire()
try:
components = urlparse.urlsplit(log_link)
base_url = '%(scheme)s://%(netloc)s' % {
'scheme': components[0],
'netloc': components[1]
}
# Takes on form (epoch time, client object)
# Least Recently Used algorithm.
client_tuple = next((tup for tup in _log_client_heap if tup[1].base_url == base_url), None)
if client_tuple is None:
client = HttpClient(base_url, LOG)
yarn_cluster = cluster.get_cluster_conf_for_job_submission()
if yarn_cluster.SECURITY_ENABLED.get():
client.set_kerberos_auth()
else:
_log_client_heap.remove(client_tuple)
client = client_tuple[1]
new_client_tuple = (time.time(), client)
if len(_log_client_heap) >= MAX_HEAP_SIZE:
heapq.heapreplace(_log_client_heap, new_client_tuple)
else:
heapq.heappush(_log_client_heap, new_client_tuple)
return client
finally:
_log_client_lock.release()
def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = is_kerberos_enabled()
self._webserver_spnego_enabled = is_webserver_spnego_enabled()
self._thread_local = threading.local()
# You can set username/password for Impala Web UI which overrides kerberos
daemon_api_pwd = \
(DAEMON_API_PASSWORD.get() if DAEMON_API_PASSWORD.get()
is not None else (DAEMON_API_PASSWORD_SCRIPT.get() if DAEMON_API_PASSWORD_SCRIPT.get()
is not None else None))
if DAEMON_API_USERNAME.get(
) is not None and daemon_api_pwd is not None:
if DAEMON_API_AUTH_SCHEME.get().lower() == 'basic':
self._client.set_basic_auth(DAEMON_API_USERNAME.get(),
daemon_api_pwd)
LOG.info(
"Using username and password for basic authentication")
else:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(),
daemon_api_pwd)
LOG.info(
'Using username and password for digest authentication')
elif self._webserver_spnego_enabled or self._security_enabled:
self._client.set_kerberos_auth()
LOG.info('Using kerberos principal for authentication')
class OptimizerApi(object):
def __init__(self,
api_url=None,
product_name=None,
product_secret=None,
ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get()):
self._api_url = (api_url or OPTIMIZER.API_URL.get()).strip('/')
self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get(
)
self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get(
)
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = resource.Resource(self._client)
def create_product(self, product_name, product_secret):
try:
data = {
'productName': product_name,
'productSecret': product_secret,
'authCode': ''
}
return self._root.post('/api/createProduct', data)
except RestException, e:
raise PopupException(e, title=_('Error while accessing Optimizer'))
def _query_store_proxy(request, path=None):
response = {'status': -1}
headers = {
'x-do-as': request.user.username,
'X-Requested-By': 'das',
'Content-Type': 'application/json; charset=UTF-8'
}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
if USE_SASL.get():
client.set_kerberos_auth()
try:
response = resource.invoke(request.method, path, request.GET.dict(),
request.body, headers)
except RestException as e:
ex_response = e.get_parent_ex().response
if ex_response is not None:
response['code'] = ex_response.status_code
response['message'] = ex_response.reason
response['content'] = ex_response.text
else:
response['message'] = 'Query store not reachable!'
response['content'] = e.message
return response
class SparkHistoryServerApi(object):
def __init__(self,
spark_hs_url,
security_enabled=False,
ssl_cert_ca_verify=False):
self._ui_url = spark_hs_url
self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "Spark History Server API at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def ui_url(self):
return self._ui_url
@property
def headers(self):
return {'Accept': _JSON_CONTENT_TYPE}
def applications(self):
return self._root.get('applications', headers=self.headers)
def application(self, app_id):
return self._root.get('applications/%(app_id)s' % {'app_id': app_id},
headers=self.headers)
def jobs(self, app_id, attempt_id):
return self._root.get('applications/%(app_id)s/%(attempt_id)s/jobs' % {
'app_id': app_id,
'attempt_id': attempt_id
},
headers=self.headers)
def stages(self, app_id, attempt_id):
return self._root.get('applications/%(app_id)s/%(attempt_id)s/stages' %
{
'app_id': app_id,
'attempt_id': attempt_id
},
headers=self.headers)
def executors(self, app_id, attempt_id):
return self._root.get(
'applications/%(app_id)s/%(attempt_id)s/executors' % {
'app_id': app_id,
'attempt_id': attempt_id
},
headers=self.headers)
class SolrApi(object):
"""
http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
"""
def __init__(self, solr_url, user):
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = SECURITY_ENABLED.get()
if self.security_enabled:
self._client.set_kerberos_auth()
self._root = resource.Resource(self._client)
def _get_params(self):
if self.security_enabled:
return (('doAs', self._user), )
return (
('user.name', DEFAULT_USER),
('doAs', self._user),
)
@classmethod
def _get_json(cls, response):
if type(response) != dict:
# Got 'plain/text' mimetype instead of 'application/json'
try:
response = json.loads(response)
except ValueError, e:
# Got some null bytes in the response
LOG.error('%s: %s' % (unicode(e), repr(response)))
response = json.loads(response.replace('\x00', ''))
return response
class OptimizerApi(object):
def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None):
self._api_url = (api_url or get_optimizer_url()).strip('/')
self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get()
self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get()
self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get()
self._email = OPTIMIZER.EMAIL.get()
self._email_password = OPTIMIZER.EMAIL_PASSWORD.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = resource.Resource(self._client)
self._token = None
def _authenticate(self, force=False):
if self._token is None or force:
self._token = self.authenticate()['token']
return self._token
def create_product(self, product_name=None, product_secret=None, authCode=None):
try:
data = {
'productName': product_name if product_name is not None else self._product_name,
'productSecret': product_secret if product_secret is not None else self._product_secret,
'authCode': authCode if authCode is not None else self._product_auth_secret
}
return self._root.post('/api/createProduct', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
except RestException, e:
raise PopupException(e, title=_('Error while accessing Optimizer'))
def __init__(self,
solr_url=None,
user=None,
security_enabled=False,
ssl_cert_ca_verify=SSL_CERT_CA_VERIFY.get()):
if solr_url is None:
solr_url = SOLR_URL.get()
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = security_enabled or SECURITY_ENABLED.get()
if self.security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
self._root = resource.Resource(self._client)
# The Kerberos handshake requires two requests in order to authenticate,
# but if our first request is a PUT/POST, it might flat-out reject the
# first request if the body is too large. So, connect here in order to get
# a cookie so future PUT/POSTs will be pre-authenticated.
if self.security_enabled:
self._root.invoke('HEAD', '/')
class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "ResourceManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def cluster(self, **kwargs):
return self._root.get('cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def apps(self, **kwargs):
return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def app(self, app_id):
return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def kill(self, app_id):
return self._root.put('cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE)
def get_log_client(log_link):
global _log_client_queue
global MAX_HEAP_SIZE
_log_client_lock.acquire()
try:
components = urlparse.urlsplit(log_link)
base_url = '%(scheme)s://%(netloc)s' % {
'scheme': components[0],
'netloc': components[1]
}
# Takes on form (epoch time, client object)
# Least Recently Used algorithm.
client_tuple = next(
(tup for tup in _log_client_heap if tup[1].base_url == base_url),
None)
if client_tuple is None:
client = HttpClient(base_url, LOG)
yarn_cluster = cluster.get_cluster_conf_for_job_submission()
if yarn_cluster.SECURITY_ENABLED.get():
client.set_kerberos_auth()
else:
_log_client_heap.remove(client_tuple)
client = client_tuple[1]
new_client_tuple = (time.time(), client)
if len(_log_client_heap) >= MAX_HEAP_SIZE:
heapq.heapreplace(_log_client_heap, new_client_tuple)
else:
heapq.heappush(_log_client_heap, new_client_tuple)
return client
finally:
_log_client_lock.release()
class PrometheusApi(object):
def __init__(self, user=None, ssl_cert_ca_verify=False):
self._api_url = '%s/%s' % (PROMETHEUS.API_URL.get().strip('/'), VERSION)
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = Resource(self._client)
def query(self, query):
try:
return self._root.get('query', {
'query': query,
})['data']
except RestException as e:
raise PrometheusApiException(e)
def range_query(self, query, start, end, step):
# e.g. /api/v1/query_range?query=up&start=2015-07-01T20:10:30.781Z&end=2015-07-01T20:11:00.781Z&step=15s
try:
return self._root.get('query_range', {
'query': query,
'start': start,
'end': end,
'step': step
})['data']
except RestException as e:
raise PrometheusApiException(e)
def __init__(self, base_url, cert_validate=True):
self._base_url = base_url
self._client = HttpClient(self._base_url,
logger=LOG,
cert_validate=cert_validate)
self._data = None
self._headers = None
self._wbuf = StringIO()
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
def __init__(self, user=None, ssl_cert_ca_verify=False):
self._api_url = '%s/%s' % (PROMETHEUS.API_URL.get().strip('/'), VERSION)
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = Resource(self._client)
def __init__(self, solr_url, user):
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = SECURITY_ENABLED.get()
if self.security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
class THttpClient(TTransportBase):
"""
HTTP transport mode for Thrift.
HTTPS and Kerberos support with Request.
e.g.
mode = THttpClient('http://hbase-thrift-v1.com:9090')
mode = THttpClient('http://hive-localhost:10001/cliservice')
"""
def __init__(self, base_url, cert_validate=True):
self._base_url = base_url
self._client = HttpClient(self._base_url,
logger=LOG,
cert_validate=cert_validate)
self._data = None
self._headers = None
self._wbuf = StringIO()
def open(self):
pass
def set_basic_auth(self, username, password):
self._client.set_basic_auth(username, password)
def set_kerberos_auth(self):
self._client.set_kerberos_auth()
def close(self):
self._headers = None
# Close session too?
def isOpen(self):
return self._client is not None
def setTimeout(self, ms):
pass
def setCustomHeaders(self, headers):
self._headers = headers
def read(self, sz):
return self._data
def write(self, buf):
self._wbuf.write(buf)
def flush(self):
if self.isOpen():
self.close()
self.open()
data = self._wbuf.getvalue()
self._wbuf = StringIO()
# POST
self._root = Resource(self._client)
self._data = self._root.post('', data=data)
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
self._thread_local = threading.local()
def __init__(self, livy_url):
self._url = posixpath.join(livy_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = SECURITY_ENABLED.get()
self._thread_local = threading.local()
if self.security_enabled:
self._client.set_kerberos_auth()
class THttpClient(TTransportBase):
"""
HTTP transport mode for Thrift.
HTTPS and Kerberos support with Request.
e.g.
mode = THttpClient('http://hbase-thrift-v1.com:9090')
mode = THttpClient('http://hive-localhost:10001/cliservice')
"""
def __init__(self, base_url, cert_validate=True):
self._base_url = base_url
self._client = HttpClient(self._base_url, logger=LOG, cert_validate=cert_validate)
self._data = None
self._headers = None
self._wbuf = StringIO()
def open(self):
pass
def set_basic_auth(self, username, password):
self._client.set_basic_auth(username, password)
def set_kerberos_auth(self):
self._client.set_kerberos_auth()
def close(self):
self._headers = None
# Close session too?
def isOpen(self):
return self._client is not None
def setTimeout(self, ms):
pass
def setCustomHeaders(self, headers):
self._headers = headers
def read(self, sz):
return self._data
def write(self, buf):
self._wbuf.write(buf)
def flush(self):
if self.isOpen():
self.close()
self.open()
data = self._wbuf.getvalue()
self._wbuf = StringIO()
# POST
self._root = Resource(self._client)
self._data = self._root.post('', data=data)
def __init__(self, url, username, language='en'):
self._url = url
self._client = HttpClient(self._url, logger=LOG)
self._root = SqoopResource(self._client)
self._language = language
self._username = username
if has_sqoop_has_security():
self._client.set_kerberos_auth()
self._security_enabled = has_sqoop_has_security()
class NavigatorApi(object):
"""
http://cloudera.github.io/navigator/apidocs/v2/index.html
"""
def __init__(self, api_url=None, user=None, password=None):
self._api_url = (api_url or NAVIGATOR.API_URL.get()).strip('/')
self._username = user or NAVIGATOR.AUTH_USERNAME.get()
self._password = password or NAVIGATOR.AUTH_PASSWORD.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_basic_auth(self._username, self._password)
self._root = resource.Resource(self._client)
self.__headers = {}
self.__params = ()
def find_entity(self, source_type, type, name, **filters):
"""
GET /api/v2/entities?query=((sourceType:<source_type>)AND(type:<type>)AND(originalName:<name>))
http://cloudera.github.io/navigator/apidocs/v2/path__v2_entities.html
"""
try:
params = self.__params
query_filters = {
'sourceType': source_type,
'type': type,
'originalName': name,
'deleted': 'false'
}
for key, value in filters.items():
query_filters[key] = value
filter_query = 'AND'.join('(%s:%s)' % (key, value) for key, value in query_filters.items())
params += (
('query', filter_query),
('offset', 0),
('limit', 2), # We are looking for single entity, so limit to 2 to check for multiple results
)
response = self._root.get('entities', headers=self.__headers, params=params)
if not response:
raise NavigatorApiException('Could not find entity with query filters: %s' % str(query_filters))
elif len(response) > 1:
raise NavigatorApiException('Found more than 1 entity with query filters: %s' % str(query_filters))
return response[0]
except RestException, e:
msg = 'Failed to find entity: %s' % str(e)
LOG.exception(msg)
raise NavigatorApiException(msg)
class MapreduceApi(object):
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, 'proxy')
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
def __str__(self):
return "MapreduceApi at %s" % (self._url,)
@property
def url(self):
return self._url
def job(self, user, job_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def counters(self, job_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/counters' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def tasks(self, job_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def job_attempts(self, job_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/jobattempts' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def conf(self, job_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/conf' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def task(self, job_id, task_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_counters(self, job_id, task_id):
app_id = job_id.replace('job', 'application')
job_id = job_id.replace('application', 'job')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/counters' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_attempts(self, job_id, task_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_attempt(self, job_id, task_id, attempt_id):
app_id = job_id.replace('job', 'application')
job_id = job_id.replace('application', 'job')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts/%(attempt_id)s' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'attempt_id': attempt_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
class NavigatorApi(object):
"""
http://cloudera.github.io/navigator/apidocs/v2/index.html
"""
def __init__(self, api_url=None, user=None, password=None):
self._api_url = (api_url or NAVIGATOR.API_URL.get()).strip('/')
self._username = user or NAVIGATOR.AUTH_USERNAME.get()
self._password = password or NAVIGATOR.AUTH_PASSWORD.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_basic_auth(self._username, self._password)
self._root = resource.Resource(self._client)
self.__headers = {}
self.__params = ()
def find_entity(self, source_type, type, name, **filters):
"""
GET /api/v2/entities?query=((sourceType:<source_type>)AND(type:<type>)AND(originalName:<name>))
http://cloudera.github.io/navigator/apidocs/v2/path__v2_entities.html
"""
try:
params = self.__params
query_filters = {
'sourceType': source_type,
'type': type,
'originalName': name,
'deleted': 'false'
}
for key, value in filters.items():
query_filters[key] = value
filter_query = 'AND'.join('(%s:%s)' % (key, value) for key, value in query_filters.items())
params += (
('query', filter_query),
('offset', 0),
('limit', 2), # We are looking for single entity, so limit to 2 to check for multiple results
)
response = self._root.get('entities', headers=self.__headers, params=params)
if not response:
raise NavigatorApiException('Could not find entity with query filters: %s' % str(query_filters))
elif len(response) > 1:
raise NavigatorApiException('Found more than 1 entity with query filters: %s' % str(query_filters))
return response[0]
except RestException, e:
msg = 'Failed to find entity: %s' % str(e)
LOG.exception(msg)
raise NavigatorApiException(msg)
def __init__(self, api_url=None, user=None, password=None):
self._api_url = (api_url or NAVIGATOR.API_URL.get()).strip('/')
self._username = user or NAVIGATOR.AUTH_USERNAME.get()
self._password = password or NAVIGATOR.AUTH_PASSWORD.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_basic_auth(self._username, self._password)
self._root = resource.Resource(self._client)
self.__headers = {}
self.__params = ()
def __init__(self, spark_hs_url, security_enabled=False, ssl_cert_ca_verify=False):
self._ui_url = spark_hs_url
self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws/%s/history' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
self._thread_local = threading.local() # To store user info
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, username, mr_url, security_enabled=False, ssl_cert_ca_verify=False):
self._user = username
self._url = posixpath.join(mr_url, 'proxy')
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
class ImpalaDaemonApi(object):
def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
# You can set username/password for Impala Web UI which overrides kerberos
if DAEMON_API_USERNAME.get() is not None and DAEMON_API_PASSWORD.get() is not None:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(), DAEMON_API_PASSWORD.get())
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
def __str__(self):
return "ImpalaDaemonApi at %s" % self._url
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
return self._thread_local.user
def set_user(self, user):
if hasattr(user, 'username'):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def get_queries(self):
params = {
'json': 'true'
}
resp = self._root.get('queries', params=params)
try:
if isinstance(resp, basestring):
return json.loads(resp)
else:
return resp
except ValueError, e:
raise ImpalaDaemonApiException('ImpalaDaemonApi did not return valid JSON: %s' % e)
def __init__(self, user=None):
self._api_url = '%s/%s' % (NAVIGATOR.API_URL.get().strip('/'), VERSION)
self._username = NAVIGATOR.AUTH_USERNAME.get()
self._password = NAVIGATOR.AUTH_PASSWORD.get()
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_basic_auth(self._username, self._password)
self._root = resource.Resource(self._client, urlencode=False) # For search_entities_interactive
self.__headers = {}
self.__params = ()
def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
# You can set username/password for Impala Web UI which overrides kerberos
if DAEMON_API_USERNAME.get() is not None and DAEMON_API_PASSWORD.get(
) is not None:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(),
DAEMON_API_PASSWORD.get())
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
class HistoryServerApi(object):
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws/%s/history' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "HistoryServerApi at %s" % (self._url,)
@property
def url(self):
return self._url
def job(self, user, job_id):
return self._root.get('mapreduce/jobs/%(job_id)s' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def counters(self, job_id):
return self._root.get('mapreduce/jobs/%(job_id)s/counters' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def conf(self, job_id):
return self._root.get('mapreduce/jobs/%(job_id)s/conf' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def job_attempts(self, job_id):
return self._root.get('mapreduce/jobs/%(job_id)s/jobattempts' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def tasks(self, job_id):
return self._root.get('mapreduce/jobs/%(job_id)s/tasks' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def task(self, job_id, task_id):
return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s' % {'job_id': job_id, 'task_id': task_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_attempts(self, job_id, task_id):
return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts' % {'job_id': job_id, 'task_id': task_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_counters(self, job_id, task_id):
job_id = job_id.replace('application', 'job')
return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/counters' % {'job_id': job_id, 'task_id': task_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_attempt(self, job_id, task_id, attempt_id):
return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts/%(attempt_id)s' % {'job_id': job_id, 'task_id': task_id, 'attempt_id': attempt_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_attempt_counters(self, job_id, task_id, attempt_id):
return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts/%(attempt_id)s/counters' % {'job_id': job_id, 'task_id': task_id, 'attempt_id': attempt_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def __init__(
self,
solr_url,
user,
security_enabled=SECURITY_ENABLED.get() if search_enabled() else SECURITY_ENABLED.default,
ssl_cert_ca_verify=SSL_CERT_CA_VERIFY.get(),
):
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = security_enabled
if self.security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
self._root = resource.Resource(self._client)
# The Kerberos handshake requires two requests in order to authenticate,
# but if our first request is a PUT/POST, it might flat-out reject the
# first request if the body is too large. So, connect here in order to get
# a cookie so future PUT/POSTs will be pre-authenticated.
if self.security_enabled:
self._root.invoke("HEAD", "/")
class SolrApi(object):
"""
http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
"""
def __init__(self, solr_url, user):
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = SECURITY_ENABLED.get()
if self.security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
def _get_params(self):
if self.security_enabled:
return (('doAs', self._user ),)
return (('user.name', DEFAULT_USER), ('doAs', self._user),)
def query(self, solr_query, hue_core):
try:
params = self._get_params() + (
('q', solr_query['q'] or EMPTY_QUERY.get()),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
params += hue_core.get_query(solr_query)
fqs = solr_query['fq'].split('|')
for fq in fqs:
if fq:
params += (('fq', fq),)
response = self._root.get('%(collection)s/select' % solr_query, params)
if type(response) != dict:
# Got 'plain/text' mimetype instead of 'application/json'
try:
response = json.loads(response)
except ValueError, e:
# Got some null bytes in the response
LOG.error('%s: %s' % (unicode(e), repr(response)))
response = json.loads(response.replace('\x00', ''))
return response
except RestException, e:
raise PopupException(e, title=_('Error while accessing Solr'))
def __init__(self, solr_url, user, security_enabled=SECURITY_ENABLED.get()):
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = security_enabled
if self.security_enabled:
self._client.set_kerberos_auth()
self._root = resource.Resource(self._client)
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
self.user = None # username actually
class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "ResourceManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def cluster(self, **kwargs):
return self._execute(self._root.get, 'cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def apps(self, **kwargs):
return self._execute(self._root.get, 'cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def app(self, app_id):
return self._execute(self._root.get, 'cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def kill(self, app_id):
return self._execute(self._root.put, 'cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE)
def _execute(self, function, *args, **kwargs):
response = function(*args, **kwargs)
# YARN-2605: Yarn does not use proper HTTP redirects when the standby RM has
# failed back to the master RM.
if isinstance(response, str) and response.startswith('This is standby RM. Redirecting to the current active RM'):
raise YarnFailoverOccurred(response)
return response
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
self._thread_local = threading.local()
def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get()):
self._api_url = (api_url or OPTIMIZER.API_URL.get()).strip('/')
self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get()
self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = resource.Resource(self._client)
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = SECURITY_ENABLED.get()
self._thread_local = threading.local()
if self.security_enabled:
self._client.set_kerberos_auth()
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'proxy')
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
# You can set username/password for Impala Web UI which overrides kerberos
if DAEMON_API_USERNAME.get() is not None and DAEMON_API_PASSWORD.get() is not None:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(), DAEMON_API_PASSWORD.get())
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
class SparkHistoryServerApi(object):
def __init__(self, spark_hs_url, security_enabled=False, ssl_cert_ca_verify=False):
self._ui_url = spark_hs_url
self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "Spark History Server API at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def ui_url(self):
return self._ui_url
@property
def headers(self):
return {'Accept': _JSON_CONTENT_TYPE}
def applications(self):
return self._root.get('applications', headers=self.headers)
def application(self, app_id):
return self._root.get('applications/%(app_id)s' % {'app_id': app_id}, headers=self.headers)
def jobs(self, app_id, attempt_id):
return self._root.get('applications/%(app_id)s/%(attempt_id)s/jobs' % {'app_id': app_id, 'attempt_id': attempt_id}, headers=self.headers)
def stages(self, app_id, attempt_id):
return self._root.get('applications/%(app_id)s/%(attempt_id)s/stages' % {'app_id': app_id, 'attempt_id': attempt_id}, headers=self.headers)
def executors(self, app_id, attempt_id):
return self._root.get('applications/%(app_id)s/%(attempt_id)s/executors' % {'app_id': app_id, 'attempt_id': attempt_id}, headers=self.headers)
def __init__(self, url, username, language='en'):
self._url = url
self._client = HttpClient(self._url, logger=LOG)
self._root = SqoopResource(self._client)
self._language = language
self._username = username
if has_sqoop_has_security():
self._client.set_kerberos_auth()
self._security_enabled = has_sqoop_has_security()
def __init__(self, api_url=None, user=None, password=None):
self._api_url = (api_url or NAVIGATOR.API_URL.get()).strip('/')
self._username = user or NAVIGATOR.AUTH_USERNAME.get()
self._password = password or NAVIGATOR.AUTH_PASSWORD.get()
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_basic_auth(self._username, self._password)
self._root = resource.Resource(self._client)
self.__headers = {}
self.__params = ()
def __init__(self, user, rm_url, security_enabled=False, ssl_cert_ca_verify=False):
self._user = user
self._url = posixpath.join(rm_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, rm_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(rm_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
self._thread_local = threading.local() # To store user info
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
