class ResourceManagerApi(object): def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=True): self._url = posixpath.join(oozie_url, "ws", _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) def __str__(self): return "NodeManagerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def containers(self): return self._root.get("node/containers", headers={"Accept": _JSON_CONTENT_TYPE}) def container(self, container_id): return self._root.get( "node/containers/%(container_id)s" % {"container_id": container_id}, headers={"Accept": _JSON_CONTENT_TYPE} )
def query_store_api(request, path=None): response = {'status': -1} if USE_PROXY.get(): content_type = 'application/json; charset=UTF-8' headers = {'X-Requested-By': 'das', 'Content-Type': content_type} client = HttpClient(QUERY_STORE.SERVER_URL.get()) resource = Resource(client) if USE_SASL.get(): client.set_kerberos_auth() try: response = resource.invoke(request.method, path, request.GET.dict(), request.body, headers) except RestException as e: ex_response = e.get_parent_ex().response response['code'] = ex_response.status_code response['message'] = ex_response.reason response['content'] = ex_response.text else: if path == 'api/query/search': filters = json.loads(request.body) resp = get_api(request.user, interface='queries-hive').apps(filters['search']) response = resp['apps'] return JsonResponse(response)
class ResourceManagerApi(object): def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False): self._url = posixpath.join(oozie_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) def __str__(self): return "ResourceManagerApi at %s" % (self._url, ) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def cluster(self, **kwargs): return self._execute(self._root.get, 'cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def apps(self, **kwargs): return self._execute(self._root.get, 'cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def app(self, app_id): return self._execute(self._root.get, 'cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def kill(self, app_id): return self._execute(self._root.put, 'cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE) def _execute(self, function, *args, **kwargs): response = function(*args, **kwargs) # YARN-2605: Yarn does not use proper HTTP redirects when the standby RM has # failed back to the master RM. if isinstance(response, str) and response.startswith( 'This is standby RM. Redirecting to the current active RM'): raise YarnFailoverOccurred(response) return response
class ResourceManagerApi(object): def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False): self._url = posixpath.join(oozie_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled self._ssl_cert_ca_verify = ssl_cert_ca_verify if self._security_enabled: self._client.set_kerberos_auth() if ssl_cert_ca_verify: self._client.set_verify(True) def __str__(self): return "ResourceManagerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def cluster(self, **kwargs): return self._root.get('cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def apps(self, **kwargs): return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def app(self, app_id): return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def kill(self, app_id): return self._root.put('cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE)
class ResourceManagerApi(object): def __init__(self, oozie_url, security_enabled=False): self._url = posixpath.join(oozie_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() def __str__(self): return "ResourceManagerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def apps(self, **kwargs): return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def app(self, app_id): return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE})
class NodeManagerApi(object): def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=True): self._url = posixpath.join(oozie_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) def __str__(self): return "NodeManagerApi at %s" % (self._url, ) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def containers(self): return self._root.get('node/containers', headers={'Accept': _JSON_CONTENT_TYPE}) def container(self, container_id): return self._root.get('node/containers/%(container_id)s' % {'container_id': container_id}, headers={'Accept': _JSON_CONTENT_TYPE})
class NavigatorApi(object): """ http://cloudera.github.io/navigator/apidocs/v3/index.html """ def __init__(self, api_url=None, user=None, password=None): self._api_url = '%s/%s' % ((api_url or NAVIGATOR.API_URL.get()).strip('/'), VERSION) self._username = user or NAVIGATOR.AUTH_USERNAME.get() self._password = password or NAVIGATOR.AUTH_PASSWORD.get() self._client = HttpClient(self._api_url, logger=LOG) self._client.set_basic_auth(self._username, self._password) self._root = resource.Resource(self._client) self.__headers = {} self.__params = () def search_entities(self, query_s, limit=100, offset=0, **filters): """ GET /api/v3/entities?query=() http://cloudera.github.io/navigator/apidocs/v3/path__v3_entities.html :param query_s: a query string of search terms (e.g. - sales quarterly); Currently the search will perform an OR boolean search for all terms (split on whitespace), against a whitelist of search_fields. TODO: support smarter boolean searching with arbitrary ordering and precedence of conditionals :param filters: TODO: IMPLEMENT ME, required to support property search """ search_fields = ('originalName', 'originalDescription', 'name', 'description', 'tags') entity_types = ('DATABASE', 'TABLE', 'PARTITION', 'FIELD', 'FILE', 'OPERATION') try: params = self.__params search_terms = [term.lower() for term in query_s.strip().split()] query_clauses = [] for term in search_terms: query_clauses.append('OR'.join(['(%s:*%s*)' % (field, term) for field in search_fields])) filter_query = '(originalName:*.*)' if search_terms: filter_query = 'OR'.join(['(%s)' % clause for clause in query_clauses]) type_filter_clause = 'OR'.join(['(%s:%s)' % ('type', entity_type) for entity_type in entity_types]) filter_query = '%sAND(%s)' % (filter_query, type_filter_clause) params += ( ('query', filter_query), ('offset', offset), ('limit', limit), ) response = self._root.get('entities', headers=self.__headers, params=params) return response except RestException, e: msg = 'Failed to search for entities with search query: %s' % query_s LOG.exception(msg) raise NavigatorApiException(msg)
class OptimizerApi(object): def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None): self._api_url = (api_url or OPTIMIZER.API_URL.get()).strip('/') self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get() self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get() self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get() self._email = OPTIMIZER.EMAIL.get() self._email_password = OPTIMIZER.EMAIL_PASSWORD.get() self._client = HttpClient(self._api_url, logger=LOG) self._client.set_verify(ssl_cert_ca_verify) self._root = resource.Resource(self._client) def create_product(self, product_name=None, product_secret=None, authCode=None): try: data = { 'productName': product_name if product_name is not None else self._product_name, 'productSecret': product_secret if product_secret is not None else self._product_secret, 'authCode': authCode if authCode is not None else self._product_auth_secret } return self._root.post('/api/createProduct', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) except RestException, e: raise PopupException(e, title=_('Error while accessing Optimizer'))
class ResourceManagerApi(object): def __init__(self, oozie_url, security_enabled=False): self._url = posixpath.join(oozie_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() def __str__(self): return "NodeManagerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def containers(self): return self._root.get('node/containers', headers={'Accept': _JSON_CONTENT_TYPE}) def container(self, container_id): return self._root.get('node/containers/%(container_id)s' % {'container_id': container_id}, headers={'Accept': _JSON_CONTENT_TYPE})
class SolrApi(object): """ http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler """ def __init__(self, solr_url): self._url = solr_url self._client = HttpClient(self._url, logger=LOG) if SECURITY_ENABLED.get(): self._client.set_kerberos_auth() self._root = Resource(self._client) def query(self, solr_query, hue_core): try: params = ( ('q', solr_query['q'] or EMPTY_QUERY.get()), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) params += hue_core.get_query(solr_query) fqs = solr_query['fq'].split('|') for fq in fqs: if fq: params += (('fq', fq),) response = self._root.get('%(collection)s/select' % solr_query, params) if type(response) != dict: # Got 'plain/text' mimetype instead of 'application/json' response = json.loads(response) return response except RestException, e: raise PopupException('Error while accessing Solr: %s' % e)
def __init__(self, user=None): self._api_url = '%s/dw' % K8S.API_URL.get().rstrip('/') self.user = user self._client = HttpClient(self._api_url, logger=LOG) self._client.set_verify(False) self._root = Resource(self._client)
class SolrApi(object): """ http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler """ def __init__(self, solr_url, user): self._url = solr_url self._user = user self._client = HttpClient(self._url, logger=LOG) self.security_enabled = SECURITY_ENABLED.get() if self.security_enabled: self._client.set_kerberos_auth() self._root = resource.Resource(self._client) def _get_params(self): if self.security_enabled: return (('doAs', self._user ),) return (('user.name', DEFAULT_USER), ('doAs', self._user),) @classmethod def _get_json(cls, response): if type(response) != dict: # Got 'plain/text' mimetype instead of 'application/json' try: response = json.loads(response) except ValueError, e: # Got some null bytes in the response LOG.error('%s: %s' % (unicode(e), repr(response))) response = json.loads(response.replace('\x00', '')) return response
def get_log_client(log_link): global _log_client_queue global MAX_HEAP_SIZE _log_client_lock.acquire() try: components = urlparse.urlsplit(log_link) base_url = '%(scheme)s://%(netloc)s' % { 'scheme': components[0], 'netloc': components[1] } # Takes on form (epoch time, client object) # Least Recently Used algorithm. client_tuple = next((tup for tup in _log_client_heap if tup[1].base_url == base_url), None) if client_tuple is None: client = HttpClient(base_url, LOG) yarn_cluster = cluster.get_cluster_conf_for_job_submission() if yarn_cluster.SECURITY_ENABLED.get(): client.set_kerberos_auth() else: _log_client_heap.remove(client_tuple) client = client_tuple[1] new_client_tuple = (time.time(), client) if len(_log_client_heap) >= MAX_HEAP_SIZE: heapq.heapreplace(_log_client_heap, new_client_tuple) else: heapq.heappush(_log_client_heap, new_client_tuple) return client finally: _log_client_lock.release()
def __init__(self, server_url): self._url = server_url self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = is_kerberos_enabled() self._webserver_spnego_enabled = is_webserver_spnego_enabled() self._thread_local = threading.local() # You can set username/password for Impala Web UI which overrides kerberos daemon_api_pwd = \ (DAEMON_API_PASSWORD.get() if DAEMON_API_PASSWORD.get() is not None else (DAEMON_API_PASSWORD_SCRIPT.get() if DAEMON_API_PASSWORD_SCRIPT.get() is not None else None)) if DAEMON_API_USERNAME.get( ) is not None and daemon_api_pwd is not None: if DAEMON_API_AUTH_SCHEME.get().lower() == 'basic': self._client.set_basic_auth(DAEMON_API_USERNAME.get(), daemon_api_pwd) LOG.info( "Using username and password for basic authentication") else: self._client.set_digest_auth(DAEMON_API_USERNAME.get(), daemon_api_pwd) LOG.info( 'Using username and password for digest authentication') elif self._webserver_spnego_enabled or self._security_enabled: self._client.set_kerberos_auth() LOG.info('Using kerberos principal for authentication')
class OptimizerApi(object): def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get()): self._api_url = (api_url or OPTIMIZER.API_URL.get()).strip('/') self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get( ) self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get( ) self._client = HttpClient(self._api_url, logger=LOG) self._client.set_verify(ssl_cert_ca_verify) self._root = resource.Resource(self._client) def create_product(self, product_name, product_secret): try: data = { 'productName': product_name, 'productSecret': product_secret, 'authCode': '' } return self._root.post('/api/createProduct', data) except RestException, e: raise PopupException(e, title=_('Error while accessing Optimizer'))
def _query_store_proxy(request, path=None): response = {'status': -1} headers = { 'x-do-as': request.user.username, 'X-Requested-By': 'das', 'Content-Type': 'application/json; charset=UTF-8' } client = HttpClient(QUERY_STORE.SERVER_URL.get()) resource = Resource(client) if USE_SASL.get(): client.set_kerberos_auth() try: response = resource.invoke(request.method, path, request.GET.dict(), request.body, headers) except RestException as e: ex_response = e.get_parent_ex().response if ex_response is not None: response['code'] = ex_response.status_code response['message'] = ex_response.reason response['content'] = ex_response.text else: response['message'] = 'Query store not reachable!' response['content'] = e.message return response
class SparkHistoryServerApi(object): def __init__(self, spark_hs_url, security_enabled=False, ssl_cert_ca_verify=False): self._ui_url = spark_hs_url self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) def __str__(self): return "Spark History Server API at %s" % (self._url, ) @property def url(self): return self._url @property def ui_url(self): return self._ui_url @property def headers(self): return {'Accept': _JSON_CONTENT_TYPE} def applications(self): return self._root.get('applications', headers=self.headers) def application(self, app_id): return self._root.get('applications/%(app_id)s' % {'app_id': app_id}, headers=self.headers) def jobs(self, app_id, attempt_id): return self._root.get('applications/%(app_id)s/%(attempt_id)s/jobs' % { 'app_id': app_id, 'attempt_id': attempt_id }, headers=self.headers) def stages(self, app_id, attempt_id): return self._root.get('applications/%(app_id)s/%(attempt_id)s/stages' % { 'app_id': app_id, 'attempt_id': attempt_id }, headers=self.headers) def executors(self, app_id, attempt_id): return self._root.get( 'applications/%(app_id)s/%(attempt_id)s/executors' % { 'app_id': app_id, 'attempt_id': attempt_id }, headers=self.headers)
class SolrApi(object): """ http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler """ def __init__(self, solr_url, user): self._url = solr_url self._user = user self._client = HttpClient(self._url, logger=LOG) self.security_enabled = SECURITY_ENABLED.get() if self.security_enabled: self._client.set_kerberos_auth() self._root = resource.Resource(self._client) def _get_params(self): if self.security_enabled: return (('doAs', self._user), ) return ( ('user.name', DEFAULT_USER), ('doAs', self._user), ) @classmethod def _get_json(cls, response): if type(response) != dict: # Got 'plain/text' mimetype instead of 'application/json' try: response = json.loads(response) except ValueError, e: # Got some null bytes in the response LOG.error('%s: %s' % (unicode(e), repr(response))) response = json.loads(response.replace('\x00', '')) return response
class OptimizerApi(object): def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get(), product_auth_secret=None): self._api_url = (api_url or get_optimizer_url()).strip('/') self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get() self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get() self._product_auth_secret = product_auth_secret if product_auth_secret else OPTIMIZER.PRODUCT_AUTH_SECRET.get() self._email = OPTIMIZER.EMAIL.get() self._email_password = OPTIMIZER.EMAIL_PASSWORD.get() self._client = HttpClient(self._api_url, logger=LOG) self._client.set_verify(ssl_cert_ca_verify) self._root = resource.Resource(self._client) self._token = None def _authenticate(self, force=False): if self._token is None or force: self._token = self.authenticate()['token'] return self._token def create_product(self, product_name=None, product_secret=None, authCode=None): try: data = { 'productName': product_name if product_name is not None else self._product_name, 'productSecret': product_secret if product_secret is not None else self._product_secret, 'authCode': authCode if authCode is not None else self._product_auth_secret } return self._root.post('/api/createProduct', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) except RestException, e: raise PopupException(e, title=_('Error while accessing Optimizer'))
def __init__(self, solr_url=None, user=None, security_enabled=False, ssl_cert_ca_verify=SSL_CERT_CA_VERIFY.get()): if solr_url is None: solr_url = SOLR_URL.get() self._url = solr_url self._user = user self._client = HttpClient(self._url, logger=LOG) self.security_enabled = security_enabled or SECURITY_ENABLED.get() if self.security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) self._root = resource.Resource(self._client) # The Kerberos handshake requires two requests in order to authenticate, # but if our first request is a PUT/POST, it might flat-out reject the # first request if the body is too large. So, connect here in order to get # a cookie so future PUT/POSTs will be pre-authenticated. if self.security_enabled: self._root.invoke('HEAD', '/')
class ResourceManagerApi(object): def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False): self._url = posixpath.join(oozie_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) def __str__(self): return "ResourceManagerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def cluster(self, **kwargs): return self._root.get('cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def apps(self, **kwargs): return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def app(self, app_id): return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def kill(self, app_id): return self._root.put('cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE)
def get_log_client(log_link): global _log_client_queue global MAX_HEAP_SIZE _log_client_lock.acquire() try: components = urlparse.urlsplit(log_link) base_url = '%(scheme)s://%(netloc)s' % { 'scheme': components[0], 'netloc': components[1] } # Takes on form (epoch time, client object) # Least Recently Used algorithm. client_tuple = next( (tup for tup in _log_client_heap if tup[1].base_url == base_url), None) if client_tuple is None: client = HttpClient(base_url, LOG) yarn_cluster = cluster.get_cluster_conf_for_job_submission() if yarn_cluster.SECURITY_ENABLED.get(): client.set_kerberos_auth() else: _log_client_heap.remove(client_tuple) client = client_tuple[1] new_client_tuple = (time.time(), client) if len(_log_client_heap) >= MAX_HEAP_SIZE: heapq.heapreplace(_log_client_heap, new_client_tuple) else: heapq.heappush(_log_client_heap, new_client_tuple) return client finally: _log_client_lock.release()
class PrometheusApi(object): def __init__(self, user=None, ssl_cert_ca_verify=False): self._api_url = '%s/%s' % (PROMETHEUS.API_URL.get().strip('/'), VERSION) self.user = user self._client = HttpClient(self._api_url, logger=LOG) self._client.set_verify(ssl_cert_ca_verify) self._root = Resource(self._client) def query(self, query): try: return self._root.get('query', { 'query': query, })['data'] except RestException as e: raise PrometheusApiException(e) def range_query(self, query, start, end, step): # e.g. /api/v1/query_range?query=up&start=2015-07-01T20:10:30.781Z&end=2015-07-01T20:11:00.781Z&step=15s try: return self._root.get('query_range', { 'query': query, 'start': start, 'end': end, 'step': step })['data'] except RestException as e: raise PrometheusApiException(e)
def __init__(self, base_url, cert_validate=True): self._base_url = base_url self._client = HttpClient(self._base_url, logger=LOG, cert_validate=cert_validate) self._data = None self._headers = None self._wbuf = StringIO()
def __init__(self, oozie_url, security_enabled=False): self._url = posixpath.join(oozie_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth()
def __init__(self, user=None, ssl_cert_ca_verify=False): self._api_url = '%s/%s' % (PROMETHEUS.API_URL.get().strip('/'), VERSION) self.user = user self._client = HttpClient(self._api_url, logger=LOG) self._client.set_verify(ssl_cert_ca_verify) self._root = Resource(self._client)
def __init__(self, solr_url, user): self._url = solr_url self._user = user self._client = HttpClient(self._url, logger=LOG) self.security_enabled = SECURITY_ENABLED.get() if self.security_enabled: self._client.set_kerberos_auth() self._root = Resource(self._client)
class THttpClient(TTransportBase): """ HTTP transport mode for Thrift. HTTPS and Kerberos support with Request. e.g. mode = THttpClient('http://hbase-thrift-v1.com:9090') mode = THttpClient('http://hive-localhost:10001/cliservice') """ def __init__(self, base_url, cert_validate=True): self._base_url = base_url self._client = HttpClient(self._base_url, logger=LOG, cert_validate=cert_validate) self._data = None self._headers = None self._wbuf = StringIO() def open(self): pass def set_basic_auth(self, username, password): self._client.set_basic_auth(username, password) def set_kerberos_auth(self): self._client.set_kerberos_auth() def close(self): self._headers = None # Close session too? def isOpen(self): return self._client is not None def setTimeout(self, ms): pass def setCustomHeaders(self, headers): self._headers = headers def read(self, sz): return self._data def write(self, buf): self._wbuf.write(buf) def flush(self): if self.isOpen(): self.close() self.open() data = self._wbuf.getvalue() self._wbuf = StringIO() # POST self._root = Resource(self._client) self._data = self._root.post('', data=data)
def __init__(self, oozie_url, security_enabled=False): self._url = posixpath.join(oozie_url, API_VERSION) self._client = HttpClient(self._url, logger=LOG) if security_enabled: self._client.set_kerberos_auth() self._root = Resource(self._client) self._security_enabled = security_enabled # To store username info self._thread_local = threading.local()
def __init__(self, livy_url): self._url = posixpath.join(livy_url) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = SECURITY_ENABLED.get() self._thread_local = threading.local() if self.security_enabled: self._client.set_kerberos_auth()
def __init__(self, url, username, language='en'): self._url = url self._client = HttpClient(self._url, logger=LOG) self._root = SqoopResource(self._client) self._language = language self._username = username if has_sqoop_has_security(): self._client.set_kerberos_auth() self._security_enabled = has_sqoop_has_security()
class NavigatorApi(object): """ http://cloudera.github.io/navigator/apidocs/v2/index.html """ def __init__(self, api_url=None, user=None, password=None): self._api_url = (api_url or NAVIGATOR.API_URL.get()).strip('/') self._username = user or NAVIGATOR.AUTH_USERNAME.get() self._password = password or NAVIGATOR.AUTH_PASSWORD.get() self._client = HttpClient(self._api_url, logger=LOG) self._client.set_basic_auth(self._username, self._password) self._root = resource.Resource(self._client) self.__headers = {} self.__params = () def find_entity(self, source_type, type, name, **filters): """ GET /api/v2/entities?query=((sourceType:<source_type>)AND(type:<type>)AND(originalName:<name>)) http://cloudera.github.io/navigator/apidocs/v2/path__v2_entities.html """ try: params = self.__params query_filters = { 'sourceType': source_type, 'type': type, 'originalName': name, 'deleted': 'false' } for key, value in filters.items(): query_filters[key] = value filter_query = 'AND'.join('(%s:%s)' % (key, value) for key, value in query_filters.items()) params += ( ('query', filter_query), ('offset', 0), ('limit', 2), # We are looking for single entity, so limit to 2 to check for multiple results ) response = self._root.get('entities', headers=self.__headers, params=params) if not response: raise NavigatorApiException('Could not find entity with query filters: %s' % str(query_filters)) elif len(response) > 1: raise NavigatorApiException('Found more than 1 entity with query filters: %s' % str(query_filters)) return response[0] except RestException, e: msg = 'Failed to find entity: %s' % str(e) LOG.exception(msg) raise NavigatorApiException(msg)
class MapreduceApi(object): def __init__(self, oozie_url, security_enabled=False): self._url = posixpath.join(oozie_url, 'proxy') self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() def __str__(self): return "MapreduceApi at %s" % (self._url,) @property def url(self): return self._url def job(self, user, job_id): app_id = job_id.replace('job', 'application') return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE}) def counters(self, job_id): app_id = job_id.replace('job', 'application') return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/counters' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE}) def tasks(self, job_id): app_id = job_id.replace('job', 'application') return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE}) def job_attempts(self, job_id): app_id = job_id.replace('job', 'application') return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/jobattempts' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE}) def conf(self, job_id): app_id = job_id.replace('job', 'application') return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/conf' % {'app_id': app_id, 'job_id': job_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE}) def task(self, job_id, task_id): app_id = job_id.replace('job', 'application') return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE}) def task_counters(self, job_id, task_id): app_id = job_id.replace('job', 'application') job_id = job_id.replace('application', 'job') return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/counters' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE}) def task_attempts(self, job_id, task_id): app_id = job_id.replace('job', 'application') return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE}) def task_attempt(self, job_id, task_id, attempt_id): app_id = job_id.replace('job', 'application') job_id = job_id.replace('application', 'job') return self._root.get('%(app_id)s/ws/%(version)s/mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts/%(attempt_id)s' % {'app_id': app_id, 'job_id': job_id, 'task_id': task_id, 'attempt_id': attempt_id, 'version': _API_VERSION}, headers={'Accept': _JSON_CONTENT_TYPE})
def __init__(self, api_url=None, user=None, password=None): self._api_url = (api_url or NAVIGATOR.API_URL.get()).strip('/') self._username = user or NAVIGATOR.AUTH_USERNAME.get() self._password = password or NAVIGATOR.AUTH_PASSWORD.get() self._client = HttpClient(self._api_url, logger=LOG) self._client.set_basic_auth(self._username, self._password) self._root = resource.Resource(self._client) self.__headers = {} self.__params = ()
def __init__(self, spark_hs_url, security_enabled=False, ssl_cert_ca_verify=False): self._ui_url = spark_hs_url self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False): self._url = posixpath.join(oozie_url, 'ws/%s/history' % _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled self._thread_local = threading.local() # To store user info if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, username, mr_url, security_enabled=False, ssl_cert_ca_verify=False): self._user = username self._url = posixpath.join(mr_url, 'proxy') self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify)
class ImpalaDaemonApi(object): def __init__(self, server_url): self._url = server_url self._client = HttpClient(self._url, logger=LOG) # You can set username/password for Impala Web UI which overrides kerberos if DAEMON_API_USERNAME.get() is not None and DAEMON_API_PASSWORD.get() is not None: self._client.set_digest_auth(DAEMON_API_USERNAME.get(), DAEMON_API_PASSWORD.get()) self._root = Resource(self._client) self._security_enabled = False self._thread_local = threading.local() def __str__(self): return "ImpalaDaemonApi at %s" % self._url @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def user(self): return self._thread_local.user def set_user(self, user): if hasattr(user, 'username'): self._thread_local.user = user.username else: self._thread_local.user = user def get_queries(self): params = { 'json': 'true' } resp = self._root.get('queries', params=params) try: if isinstance(resp, basestring): return json.loads(resp) else: return resp except ValueError, e: raise ImpalaDaemonApiException('ImpalaDaemonApi did not return valid JSON: %s' % e)
def __init__(self, user=None): self._api_url = '%s/%s' % (NAVIGATOR.API_URL.get().strip('/'), VERSION) self._username = NAVIGATOR.AUTH_USERNAME.get() self._password = NAVIGATOR.AUTH_PASSWORD.get() self.user = user self._client = HttpClient(self._api_url, logger=LOG) self._client.set_basic_auth(self._username, self._password) self._root = resource.Resource(self._client, urlencode=False) # For search_entities_interactive self.__headers = {} self.__params = ()
def __init__(self, server_url): self._url = server_url self._client = HttpClient(self._url, logger=LOG) # You can set username/password for Impala Web UI which overrides kerberos if DAEMON_API_USERNAME.get() is not None and DAEMON_API_PASSWORD.get( ) is not None: self._client.set_digest_auth(DAEMON_API_USERNAME.get(), DAEMON_API_PASSWORD.get()) self._root = Resource(self._client) self._security_enabled = False self._thread_local = threading.local()
class HistoryServerApi(object): def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False): self._url = posixpath.join(oozie_url, 'ws/%s/history' % _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) def __str__(self): return "HistoryServerApi at %s" % (self._url,) @property def url(self): return self._url def job(self, user, job_id): return self._root.get('mapreduce/jobs/%(job_id)s' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def counters(self, job_id): return self._root.get('mapreduce/jobs/%(job_id)s/counters' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def conf(self, job_id): return self._root.get('mapreduce/jobs/%(job_id)s/conf' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def job_attempts(self, job_id): return self._root.get('mapreduce/jobs/%(job_id)s/jobattempts' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def tasks(self, job_id): return self._root.get('mapreduce/jobs/%(job_id)s/tasks' % {'job_id': job_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def task(self, job_id, task_id): return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s' % {'job_id': job_id, 'task_id': task_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def task_attempts(self, job_id, task_id): return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts' % {'job_id': job_id, 'task_id': task_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def task_counters(self, job_id, task_id): job_id = job_id.replace('application', 'job') return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/counters' % {'job_id': job_id, 'task_id': task_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def task_attempt(self, job_id, task_id, attempt_id): return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts/%(attempt_id)s' % {'job_id': job_id, 'task_id': task_id, 'attempt_id': attempt_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def task_attempt_counters(self, job_id, task_id, attempt_id): return self._root.get('mapreduce/jobs/%(job_id)s/tasks/%(task_id)s/attempts/%(attempt_id)s/counters' % {'job_id': job_id, 'task_id': task_id, 'attempt_id': attempt_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def __init__( self, solr_url, user, security_enabled=SECURITY_ENABLED.get() if search_enabled() else SECURITY_ENABLED.default, ssl_cert_ca_verify=SSL_CERT_CA_VERIFY.get(), ): self._url = solr_url self._user = user self._client = HttpClient(self._url, logger=LOG) self.security_enabled = security_enabled if self.security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) self._root = resource.Resource(self._client) # The Kerberos handshake requires two requests in order to authenticate, # but if our first request is a PUT/POST, it might flat-out reject the # first request if the body is too large. So, connect here in order to get # a cookie so future PUT/POSTs will be pre-authenticated. if self.security_enabled: self._root.invoke("HEAD", "/")
class SolrApi(object): """ http://wiki.apache.org/solr/CoreAdmin#CoreAdminHandler """ def __init__(self, solr_url, user): self._url = solr_url self._user = user self._client = HttpClient(self._url, logger=LOG) self.security_enabled = SECURITY_ENABLED.get() if self.security_enabled: self._client.set_kerberos_auth() self._root = Resource(self._client) def _get_params(self): if self.security_enabled: return (('doAs', self._user ),) return (('user.name', DEFAULT_USER), ('doAs', self._user),) def query(self, solr_query, hue_core): try: params = self._get_params() + ( ('q', solr_query['q'] or EMPTY_QUERY.get()), ('wt', 'json'), ('rows', solr_query['rows']), ('start', solr_query['start']), ) params += hue_core.get_query(solr_query) fqs = solr_query['fq'].split('|') for fq in fqs: if fq: params += (('fq', fq),) response = self._root.get('%(collection)s/select' % solr_query, params) if type(response) != dict: # Got 'plain/text' mimetype instead of 'application/json' try: response = json.loads(response) except ValueError, e: # Got some null bytes in the response LOG.error('%s: %s' % (unicode(e), repr(response))) response = json.loads(response.replace('\x00', '')) return response except RestException, e: raise PopupException(e, title=_('Error while accessing Solr'))
def __init__(self, solr_url, user, security_enabled=SECURITY_ENABLED.get()): self._url = solr_url self._user = user self._client = HttpClient(self._url, logger=LOG) self.security_enabled = security_enabled if self.security_enabled: self._client.set_kerberos_auth() self._root = resource.Resource(self._client)
def __init__(self, oozie_url, security_enabled=False): self._url = posixpath.join(oozie_url, API_VERSION) self._client = HttpClient(self._url, logger=LOG) if security_enabled: self._client.set_kerberos_auth() self._root = Resource(self._client) self._security_enabled = security_enabled self.user = None # username actually
class ResourceManagerApi(object): def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False): self._url = posixpath.join(oozie_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) def __str__(self): return "ResourceManagerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def cluster(self, **kwargs): return self._execute(self._root.get, 'cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def apps(self, **kwargs): return self._execute(self._root.get, 'cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def app(self, app_id): return self._execute(self._root.get, 'cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def kill(self, app_id): return self._execute(self._root.put, 'cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE) def _execute(self, function, *args, **kwargs): response = function(*args, **kwargs) # YARN-2605: Yarn does not use proper HTTP redirects when the standby RM has # failed back to the master RM. if isinstance(response, str) and response.startswith('This is standby RM. Redirecting to the current active RM'): raise YarnFailoverOccurred(response) return response
def __init__(self, api_url=None, product_name=None, product_secret=None, ssl_cert_ca_verify=OPTIMIZER.SSL_CERT_CA_VERIFY.get()): self._api_url = (api_url or OPTIMIZER.API_URL.get()).strip('/') self._product_name = product_name if product_name else OPTIMIZER.PRODUCT_NAME.get() self._product_secret = product_secret if product_secret else OPTIMIZER.PRODUCT_SECRET.get() self._client = HttpClient(self._api_url, logger=LOG) self._client.set_verify(ssl_cert_ca_verify) self._root = resource.Resource(self._client)
def __init__(self, oozie_url): self._url = posixpath.join(oozie_url) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = SECURITY_ENABLED.get() self._thread_local = threading.local() if self.security_enabled: self._client.set_kerberos_auth()
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False): self._url = posixpath.join(oozie_url, 'proxy') self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, server_url): self._url = server_url self._client = HttpClient(self._url, logger=LOG) # You can set username/password for Impala Web UI which overrides kerberos if DAEMON_API_USERNAME.get() is not None and DAEMON_API_PASSWORD.get() is not None: self._client.set_digest_auth(DAEMON_API_USERNAME.get(), DAEMON_API_PASSWORD.get()) self._root = Resource(self._client) self._security_enabled = False self._thread_local = threading.local()
class SparkHistoryServerApi(object): def __init__(self, spark_hs_url, security_enabled=False, ssl_cert_ca_verify=False): self._ui_url = spark_hs_url self._url = posixpath.join(spark_hs_url, 'api/%s/' % _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) def __str__(self): return "Spark History Server API at %s" % (self._url,) @property def url(self): return self._url @property def ui_url(self): return self._ui_url @property def headers(self): return {'Accept': _JSON_CONTENT_TYPE} def applications(self): return self._root.get('applications', headers=self.headers) def application(self, app_id): return self._root.get('applications/%(app_id)s' % {'app_id': app_id}, headers=self.headers) def jobs(self, app_id, attempt_id): return self._root.get('applications/%(app_id)s/%(attempt_id)s/jobs' % {'app_id': app_id, 'attempt_id': attempt_id}, headers=self.headers) def stages(self, app_id, attempt_id): return self._root.get('applications/%(app_id)s/%(attempt_id)s/stages' % {'app_id': app_id, 'attempt_id': attempt_id}, headers=self.headers) def executors(self, app_id, attempt_id): return self._root.get('applications/%(app_id)s/%(attempt_id)s/executors' % {'app_id': app_id, 'attempt_id': attempt_id}, headers=self.headers)
def __init__(self, user, rm_url, security_enabled=False, ssl_cert_ca_verify=False): self._user = user self._url = posixpath.join(rm_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify)
def __init__(self, rm_url, security_enabled=False, ssl_cert_ca_verify=False): self._url = posixpath.join(rm_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled self._thread_local = threading.local() # To store user info if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify)