def get_task_log(self, offset=0):
logs = []
attempt = self.task.job.job_attempts['jobAttempt'][-1]
log_link = attempt['logsLink']
# Get MR task logs
if self.assignedContainerId:
log_link = log_link.replace(attempt['containerId'], self.assignedContainerId)
if hasattr(self, 'nodeHttpAddress'):
log_link = log_link.replace(attempt['nodeHttpAddress'].split(':')[0], self.nodeHttpAddress.split(':')[0])
for name in ('stdout', 'stderr', 'syslog'):
link = '/%s/' % name
params = {}
if int(offset) >= 0:
params['start'] = offset
try:
log_link = re.sub('job_[^/]+', self.id, log_link)
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
response = root.get(link, params=params)
log = html.fromstring(response).xpath('/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s') % e
logs.append(log)
class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
def __str__(self):
return "NodeManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def containers(self):
return self._root.get('node/containers', headers={'Accept': _JSON_CONTENT_TYPE})
def container(self, container_id):
return self._root.get('node/containers/%(container_id)s' % {'container_id': container_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def get_task_log(self, offset=0):
logs = []
attempt = self.task.job.job_attempts['jobAttempt'][-1]
log_link = attempt['logsLink']
# Get MR task logs
if self.assignedContainerId:
log_link = log_link.replace(attempt['containerId'],
self.assignedContainerId)
if hasattr(self, 'nodeHttpAddress'):
log_link = log_link.replace(
attempt['nodeHttpAddress'].split(':')[0],
self.nodeHttpAddress.split(':')[0])
for name in ('stdout', 'stderr', 'syslog'):
link = '/%s/' % name
params = {}
if int(offset) >= 0:
params['start'] = offset
try:
log_link = re.sub('job_[^/]+', self.id, log_link)
root = Resource(get_log_client(log_link),
urlparse.urlsplit(log_link)[2],
urlencode=False)
response = root.get(link, params=params)
log = html.fromstring(response).xpath(
'/html/body/table/tbody/tr/td[2]')[0].text_content()
except Exception, e:
log = _('Failed to retrieve log: %s') % e
logs.append(log)
def get_task_log(self, offset=0):
logs = []
attempt = self.task.job.job_attempts["jobAttempt"][-1]
log_link = attempt["logsLink"]
# Get MR task logs
if self.assignedContainerId:
log_link = log_link.replace(attempt["containerId"], self.assignedContainerId)
if hasattr(self, "nodeHttpAddress"):
log_link = log_link.replace(attempt["nodeHttpAddress"].split(":")[0], self.nodeHttpAddress.split(":")[0])
for name in ("stdout", "stderr", "syslog"):
link = "/%s/" % name
params = {}
if int(offset) >= 0:
params["start"] = offset
try:
log_link = re.sub("job_[^/]+", self.id, log_link)
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
response = root.get(link, params=params)
log = html.fromstring(response).xpath("/html/body/table/tbody/tr/td[2]")[0].text_content()
except Exception, e:
log = _("Failed to retrieve log: %s") % e
logs.append(log)
def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = is_kerberos_enabled()
self._webserver_spnego_enabled = is_webserver_spnego_enabled()
self._thread_local = threading.local()
# You can set username/password for Impala Web UI which overrides kerberos
daemon_api_pwd = \
(DAEMON_API_PASSWORD.get() if DAEMON_API_PASSWORD.get()
is not None else (DAEMON_API_PASSWORD_SCRIPT.get() if DAEMON_API_PASSWORD_SCRIPT.get()
is not None else None))
if DAEMON_API_USERNAME.get(
) is not None and daemon_api_pwd is not None:
if DAEMON_API_AUTH_SCHEME.get().lower() == 'basic':
self._client.set_basic_auth(DAEMON_API_USERNAME.get(),
daemon_api_pwd)
LOG.info(
"Using username and password for basic authentication")
else:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(),
daemon_api_pwd)
LOG.info(
'Using username and password for digest authentication')
elif self._webserver_spnego_enabled or self._security_enabled:
self._client.set_kerberos_auth()
LOG.info('Using kerberos principal for authentication')
class NodeManagerApi(object):
def __init__(self,
oozie_url,
security_enabled=False,
ssl_cert_ca_verify=True):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "NodeManagerApi at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def containers(self):
return self._root.get('node/containers',
headers={'Accept': _JSON_CONTENT_TYPE})
def container(self, container_id):
return self._root.get('node/containers/%(container_id)s' %
{'container_id': container_id},
headers={'Accept': _JSON_CONTENT_TYPE})
class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
self._ssl_cert_ca_verify = ssl_cert_ca_verify
if self._security_enabled:
self._client.set_kerberos_auth()
if ssl_cert_ca_verify:
self._client.set_verify(True)
def __str__(self):
return "ResourceManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def cluster(self, **kwargs):
return self._root.get('cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def apps(self, **kwargs):
return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def app(self, app_id):
return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def kill(self, app_id):
return self._root.put('cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE)
def flush(self):
data = self._wbuf.getvalue()
self._wbuf = buffer_writer()
# POST
self._root = Resource(self._client)
self._data = self._root.post('', data=data, headers=self._headers)
def query_store_api(request, path=None):
response = {'status': -1}
if USE_PROXY.get():
content_type = 'application/json; charset=UTF-8'
headers = {'X-Requested-By': 'das', 'Content-Type': content_type}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
if USE_SASL.get():
client.set_kerberos_auth()
try:
response = resource.invoke(request.method, path,
request.GET.dict(), request.body,
headers)
except RestException as e:
ex_response = e.get_parent_ex().response
response['code'] = ex_response.status_code
response['message'] = ex_response.reason
response['content'] = ex_response.text
else:
if path == 'api/query/search':
filters = json.loads(request.body)
resp = get_api(request.user,
interface='queries-hive').apps(filters['search'])
response = resp['apps']
return JsonResponse(response)
class SparkHistoryServerApi(object):
def __init__(self,
spark_hs_url,
security_enabled=False,
ssl_cert_ca_verify=False):
self._ui_url = spark_hs_url
self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "Spark History Server API at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def ui_url(self):
return self._ui_url
@property
def headers(self):
return {'Accept': _JSON_CONTENT_TYPE}
def applications(self):
return self._root.get('applications', headers=self.headers)
def application(self, app_id):
return self._root.get('applications/%(app_id)s' % {'app_id': app_id},
headers=self.headers)
def jobs(self, app_id, attempt_id):
return self._root.get('applications/%(app_id)s/%(attempt_id)s/jobs' % {
'app_id': app_id,
'attempt_id': attempt_id
},
headers=self.headers)
def stages(self, app_id, attempt_id):
return self._root.get('applications/%(app_id)s/%(attempt_id)s/stages' %
{
'app_id': app_id,
'attempt_id': attempt_id
},
headers=self.headers)
def executors(self, app_id, attempt_id):
return self._root.get(
'applications/%(app_id)s/%(attempt_id)s/executors' % {
'app_id': app_id,
'attempt_id': attempt_id
},
headers=self.headers)
class PrometheusApi(object):
def __init__(self, user=None, ssl_cert_ca_verify=False):
self._api_url = '%s/%s' % (PROMETHEUS.API_URL.get().strip('/'), VERSION)
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = Resource(self._client)
def query(self, query):
try:
return self._root.get('query', {
'query': query,
})['data']
except RestException as e:
raise PrometheusApiException(e)
def range_query(self, query, start, end, step):
# e.g. /api/v1/query_range?query=up&start=2015-07-01T20:10:30.781Z&end=2015-07-01T20:11:00.781Z&step=15s
try:
return self._root.get('query_range', {
'query': query,
'start': start,
'end': end,
'step': step
})['data']
except RestException as e:
raise PrometheusApiException(e)
class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=True):
self._url = posixpath.join(oozie_url, "ws", _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "NodeManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def containers(self):
return self._root.get("node/containers", headers={"Accept": _JSON_CONTENT_TYPE})
def container(self, container_id):
return self._root.get(
"node/containers/%(container_id)s" % {"container_id": container_id}, headers={"Accept": _JSON_CONTENT_TYPE}
)
class SchemaRegistryApi(object):
"""
https://github.com/confluentinc/schema-registry
"""
def __init__(self,
user=None,
security_enabled=False,
ssl_cert_ca_verify=False):
self._api_url = KAFKA.SCHEMA_REGISTRY_API_URL.get().strip(
'/') if KAFKA.SCHEMA_REGISTRY_API_URL.get() else ''
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._root = Resource(self._client)
def subjects(self):
try:
response = self._root.get('subjects')
return json.loads(response)
except RestException as e:
raise KafkaApiException(e)
def subject(self, name):
try:
response = self._root.get('subjects/%s/versions/latest' % name)
return json.loads(response)
except RestException as e:
raise KafkaApiException(e)
def __init__(self, user=None):
self._api_url = '%s/dw' % K8S.API_URL.get().rstrip('/')
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(False)
self._root = Resource(self._client)
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = False
# To store user info
self._thread_local = threading.local()
class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "ResourceManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def cluster(self, **kwargs):
return self._root.get('cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def apps(self, **kwargs):
return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def app(self, app_id):
return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def kill(self, app_id):
return self._root.put('cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE)
def _query_store_proxy(request, path=None):
response = {'status': -1}
headers = {
'x-do-as': request.user.username,
'X-Requested-By': 'das',
'Content-Type': 'application/json; charset=UTF-8'
}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
if USE_SASL.get():
client.set_kerberos_auth()
try:
response = resource.invoke(request.method, path, request.GET.dict(),
request.body, headers)
except RestException as e:
ex_response = e.get_parent_ex().response
if ex_response is not None:
response['code'] = ex_response.status_code
response['message'] = ex_response.reason
response['content'] = ex_response.text
else:
response['message'] = 'Query store not reachable!'
response['content'] = e.message
return response
class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
def __str__(self):
return "ResourceManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def apps(self, **kwargs):
return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def app(self, app_id):
return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def flush(self):
data = self._wbuf.getvalue()
self._wbuf = string_io()
# POST
self._root = Resource(self._client)
self._data = self._root.post('', data=data, headers=self._headers)
def __init__(self, user=None, ssl_cert_ca_verify=False):
self._api_url = '%s/%s' % (PROMETHEUS.API_URL.get().strip('/'), VERSION)
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._client.set_verify(ssl_cert_ca_verify)
self._root = Resource(self._client)
def get_kafka_topics(self, broker_host):
try:
client = HttpClient('http://%s:24042' % broker_host, logger=LOG)
root = Resource(client)
return root.get('/api/topics')
except RestException as e:
raise ManagerApiException(e)
def __init__(self, solr_url, user):
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = SECURITY_ENABLED.get()
if self.security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
self._thread_local = threading.local()
def __init__(self, livy_url):
self._url = posixpath.join(livy_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = SECURITY_ENABLED.get()
self._thread_local = threading.local()
if self.security_enabled:
self._client.set_kerberos_auth()
def __init__(self, user, api_url=None, auth_key=None, auth_key_secret=None, tenant_id='hue'):
self.user = user
self._tenant_id = tenant_id
self._api_url = (api_url or get_optimizer_url()).strip('/')
self._client = HttpClient(self._api_url, logger=LOG)
self._root = Resource(self._client)
self._api = MockApiLib()
def __init__(self,
user=None,
security_enabled=False,
ssl_cert_ca_verify=False):
self._api_url = KAFKA.SCHEMA_REGISTRY_API_URL.get().strip(
'/') if KAFKA.SCHEMA_REGISTRY_API_URL.get() else ''
self.user = user
self._client = HttpClient(self._api_url, logger=LOG)
self._root = Resource(self._client)
def query_store_proxy(request, path=None):
content_type = 'application/json; charset=UTF-8'
headers = {'X-Requested-By': 'das', 'Content-Type': content_type}
client = HttpClient(QUERY_STORE.SERVER_URL.get())
resource = Resource(client)
return JsonResponse(
resource.invoke(request.method, path, request.GET.dict(), request.body,
headers))
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws/%s/history' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
self._thread_local = threading.local() # To store user info
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, spark_hs_url, security_enabled=False, ssl_cert_ca_verify=False):
self._ui_url = spark_hs_url
self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
class JobServerApi(object):
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = False
# To store user info
self._thread_local = threading.local()
def __str__(self):
return "JobServerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
return self._thread_local.user
def setuser(self, user):
if hasattr(user, "username"):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def get_status(self, **kwargs):
return self._root.get("healthz", params=kwargs, headers={"Accept": _TEXT_CONTENT_TYPE})
def submit_job(self, appName, classPath, data, context=None, sync=False):
params = {"appName": appName, "classPath": classPath, "sync": sync}
if context:
params["context"] = context
return self._root.post("jobs" % params, params=params, data=data, contenttype=_BINARY_CONTENT_TYPE)
def job(self, job_id):
return self._root.get("jobs/%s" % job_id, headers={"Accept": _JSON_CONTENT_TYPE})
def jobs(self, **kwargs):
return self._root.get("jobs", params=kwargs, headers={"Accept": _JSON_CONTENT_TYPE})
def create_context(self, name, **kwargs):
return self._root.post("contexts/%s" % name, params=kwargs, contenttype=_BINARY_CONTENT_TYPE)
def contexts(self, **kwargs):
return self._root.get("contexts", params=kwargs, headers={"Accept": _JSON_CONTENT_TYPE})
def delete_context(self, name, **kwargs):
return self._root.delete("contexts/%s" % name)
def upload_jar(self, app_name, data):
return self._root.post("jars/%s" % app_name, data=data, contenttype=_BINARY_CONTENT_TYPE)
def jars(self, **kwargs):
return self._root.get("jars", params=kwargs, headers={"Accept": _JSON_CONTENT_TYPE})
def __init__(self, username, mr_url, security_enabled=False, ssl_cert_ca_verify=False):
self._user = username
self._url = posixpath.join(mr_url, 'proxy')
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def flush(self):
if self.isOpen():
self.close()
self.open()
data = self._wbuf.getvalue()
self._wbuf = StringIO()
# POST
self._root = Resource(self._client)
self._data = self._root.post('', data=data)
def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
# You can set username/password for Impala Web UI which overrides kerberos
if DAEMON_API_USERNAME.get() is not None and DAEMON_API_PASSWORD.get(
) is not None:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(),
DAEMON_API_PASSWORD.get())
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
class JobServerApi(object):
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
def __str__(self):
return "JobServerApi at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
return self._thread_local.user
def setuser(self, user):
if hasattr(user, 'username'):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def create_session(self, **kwargs):
return self._root.post('sessions',
data=json.dumps(kwargs),
contenttype='application/json')
def submit_statement(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/statements' % uuid,
data=json.dumps(data),
contenttype=_JSON_CONTENT_TYPE)
def inspect(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/inspect' % uuid,
data=json.dumps(data),
contenttype=_JSON_CONTENT_TYPE)
def fetch_data(self, session, statement):
return self._root.get('sessions/%s/statements/%s' %
(session, statement))
def cancel(self, session):
return self._root.post('sessions/%s/interrupt' % session)
def __init__(self, oozie_url, user, security_enabled=False, api_version=API_VERSION):
self._url = posixpath.join(oozie_url, api_version)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
if hasattr(user, 'username'):
self.user = user.username
else:
self.user = user
self.api_version = api_version
def get_log_list(self):
log_link, user = self.get_log_link()
if not log_link:
return []
params = {
'doAs': user
}
log_link = re.sub('job_[^/]+', str(self.id), log_link)
root = Resource(get_log_client(log_link), urlparse.urlsplit(log_link)[2], urlencode=False)
response = root.get('/', params=params)
links = html.fromstring(response, parser=html.HTMLParser()).xpath('/html/body/table/tbody/tr/td[2]//a/@href')
parsed_links = map(lambda x: urlparse.urlsplit(x), links)
return map(lambda x: x and len(x) >= 2 and x[2].split('/')[-2] or '', parsed_links)
class MapreduceApi(object):
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, 'proxy')
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
def __str__(self):
return "MapreduceApi at %s" % (self._url,)
@property
def url(self):
return self._url
def job(self, user, job_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def counters(self, job_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/counters' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def tasks(self, job_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def job_attempts(self, job_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/jobattempts' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def conf(self, job_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/conf' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def task(self, job_id, task_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_counters(self, job_id, task_id):
app_id = job_id.replace('job', 'application')
job_id = job_id.replace('application', 'job')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/counters' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_attempts(self, job_id, task_id):
app_id = job_id.replace('job', 'application')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_attempt(self, job_id, task_id, attempt_id):
app_id = job_id.replace('job', 'application')
job_id = job_id.replace('application', 'job')
return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts/%(attempt_id)s' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'attempt_id': attempt_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
class HistoryServerApi(object):
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws/%s/history' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "HistoryServerApi at %s" % (self._url,)
@property
def url(self):
return self._url
def job(self, user, job_id):
return self._root.get('mapreduce/jobs/%(job_id)s' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def counters(self, job_id):
return self._root.get('mapreduce/jobs/%(job_id)s/counters' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def conf(self, job_id):
return self._root.get('mapreduce/jobs/%(job_id)s/conf' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def job_attempts(self, job_id):
return self._root.get('mapreduce/jobs/%(job_id)s/jobattempts' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def tasks(self, job_id):
return self._root.get('mapreduce/jobs/%(job_id)s/tasks' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def task(self, job_id, task_id):
return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s' % {'job_id': job_id, 'task_id': task_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_attempts(self, job_id, task_id):
return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts' % {'job_id': job_id, 'task_id': task_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_counters(self, job_id, task_id):
job_id = job_id.replace('application', 'job')
return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/counters' % {'job_id': job_id, 'task_id': task_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_attempt(self, job_id, task_id, attempt_id):
return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts/%(attempt_id)s' % {'job_id': job_id, 'task_id': task_id, 'attempt_id': attempt_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def task_attempt_counters(self, job_id, task_id, attempt_id):
return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts/%(attempt_id)s/counters' % {'job_id': job_id, 'task_id': task_id, 'attempt_id': attempt_id}, headers={'Accept': _JSON_CONTENT_TYPE})
class SolrApi(object):
"""
http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
"""
def __init__(self, solr_url):
self._url = solr_url
self._client = HttpClient(self._url, logger=LOG)
if SECURITY_ENABLED.get():
self._client.set_kerberos_auth()
self._root = Resource(self._client)
def query(self, solr_query, hue_core):
try:
params = (
('q', solr_query['q'] or EMPTY_QUERY.get()),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
params += hue_core.get_query(solr_query)
fqs = solr_query['fq'].split('|')
for fq in fqs:
if fq:
params += (('fq', fq),)
response = self._root.get('%(collection)s/select' % solr_query, params)
if type(response) != dict:
# Got 'plain/text' mimetype instead of 'application/json'
response = json.loads(response)
return response
except RestException, e:
raise PopupException('Error while accessing Solr: %s' % e)
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = False
# To store user info
self._thread_local = threading.local()
def flush(self):
data = self._wbuf.getvalue()
self._wbuf = StringIO()
# POST
self._root = Resource(self._client)
self._data = self._root.post('', data=data, headers=self._headers)
class SolrApi(object):
"""
http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
"""
def __init__(self, solr_url):
self._url = solr_url
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
def query(self, solr_query, hue_core):
try:
params = (
("q", solr_query["q"] or EMPTY_QUERY.get()),
("wt", "json"),
("rows", solr_query["rows"]),
("start", solr_query["start"]),
)
params += hue_core.get_query(solr_query)
fqs = solr_query["fq"].split("|")
for fq in fqs:
if fq:
params += (("fq", fq),)
response = self._root.get("%(collection)s/select" % solr_query, params)
if type(response) != dict:
# Got 'plain/text' mimetype instead of 'application/json'
response = json.loads(response)
return response
except RestException, e:
raise PopupException("Error while accessing Solr: %s" % e)
class SolrApi(object):
"""
http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler
"""
def __init__(self, solr_url):
self._url = solr_url
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
def query(self, solr_query, hue_core):
try:
params = (
('q', solr_query['q'] or EMPTY_QUERY.get()),
('wt', 'json'),
('rows', solr_query['rows']),
('start', solr_query['start']),
)
params += hue_core.get_query()
fqs = solr_query['fq'].split('|')
for fq in fqs:
if fq:
params += (('fq', fq),)
response = self._root.get('%(collection)s/select' % solr_query, params)
return json.loads(response)
except RestException, e:
raise PopupException('Error while accessing Solr: %s' % e)
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
self.user = None # username actually
def __init__(self, solr_url, user):
self._url = solr_url
self._user = user
self._client = HttpClient(self._url, logger=LOG)
self.security_enabled = SECURITY_ENABLED.get()
if self.security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = SECURITY_ENABLED.get()
self._thread_local = threading.local()
if self.security_enabled:
self._client.set_kerberos_auth()
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
self._thread_local = threading.local()
class THttpClient(TTransportBase):
"""
HTTP transport mode for Thrift.
HTTPS and Kerberos support with Request.
e.g.
mode = THttpClient('http://hbase-thrift-v1.com:9090')
mode = THttpClient('http://hive-localhost:10001/cliservice')
"""
def __init__(self, base_url, cert_validate=True):
self._base_url = base_url
self._client = HttpClient(self._base_url, logger=LOG, cert_validate=cert_validate)
self._data = None
self._headers = None
self._wbuf = StringIO()
def open(self):
pass
def set_basic_auth(self, username, password):
self._client.set_basic_auth(username, password)
def set_kerberos_auth(self):
self._client.set_kerberos_auth()
def close(self):
self._headers = None
# Close session too?
def isOpen(self):
return self._client is not None
def setTimeout(self, ms):
pass
def setCustomHeaders(self, headers):
self._headers = headers
def read(self, sz):
return self._data
def write(self, buf):
self._wbuf.write(buf)
def flush(self):
if self.isOpen():
self.close()
self.open()
data = self._wbuf.getvalue()
self._wbuf = StringIO()
# POST
self._root = Resource(self._client)
self._data = self._root.post('', data=data)
class SparkHistoryServerApi(object):
def __init__(self, spark_hs_url, security_enabled=False, ssl_cert_ca_verify=False):
self._ui_url = spark_hs_url
self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "Spark History Server API at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def ui_url(self):
return self._ui_url
@property
def headers(self):
return {'Accept': _JSON_CONTENT_TYPE}
def applications(self):
return self._root.get('applications', headers=self.headers)
def application(self, app_id):
return self._root.get('applications/%(app_id)s' % {'app_id': app_id}, headers=self.headers)
def jobs(self, app_id, attempt_id):
return self._root.get('applications/%(app_id)s/%(attempt_id)s/jobs' % {'app_id': app_id, 'attempt_id': attempt_id}, headers=self.headers)
def stages(self, app_id, attempt_id):
return self._root.get('applications/%(app_id)s/%(attempt_id)s/stages' % {'app_id': app_id, 'attempt_id': attempt_id}, headers=self.headers)
def executors(self, app_id, attempt_id):
return self._root.get('applications/%(app_id)s/%(attempt_id)s/executors' % {'app_id': app_id, 'attempt_id': attempt_id}, headers=self.headers)
def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
# You can set username/password for Impala Web UI which overrides kerberos
if DAEMON_API_USERNAME.get() is not None and DAEMON_API_PASSWORD.get() is not None:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(), DAEMON_API_PASSWORD.get())
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'proxy')
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
self._ssl_cert_ca_verify = ssl_cert_ca_verify
if self._security_enabled:
self._client.set_kerberos_auth()
if ssl_cert_ca_verify:
self._client.set_verify(True)
def __init__(self, mr_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(mr_url, 'proxy')
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
self._thread_local = threading.local() # To store user info
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def flush(self):
if self.isOpen():
self.close()
self.open()
data = self._wbuf.getvalue()
self._wbuf = StringIO()
# POST
self._root = Resource(self._client)
self._data = self._root.post('', data=data)
def __init__(self, spark_hs_url, security_enabled=False, ssl_cert_ca_verify=False):
self._ui_url = spark_hs_url
self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, livy_url):
self._url = posixpath.join(livy_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = SECURITY_ENABLED.get()
self._thread_local = threading.local()
if self.security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(SSL_CERT_CA_VERIFY.get())
class ImpalaDaemonApi(object):
def __init__(self, server_url):
self._url = server_url
self._client = HttpClient(self._url, logger=LOG)
# You can set username/password for Impala Web UI which overrides kerberos
if DAEMON_API_USERNAME.get() is not None and DAEMON_API_PASSWORD.get() is not None:
self._client.set_digest_auth(DAEMON_API_USERNAME.get(), DAEMON_API_PASSWORD.get())
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
def __str__(self):
return "ImpalaDaemonApi at %s" % self._url
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
return self._thread_local.user
def set_user(self, user):
if hasattr(user, 'username'):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def get_queries(self):
params = {
'json': 'true'
}
resp = self._root.get('queries', params=params)
try:
if isinstance(resp, basestring):
return json.loads(resp)
else:
return resp
except ValueError, e:
raise ImpalaDaemonApiException('ImpalaDaemonApi did not return valid JSON: %s' % e)