class JobServerApi(object): def __init__(self, oozie_url): self._url = posixpath.join(oozie_url) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = False # To store user info self._thread_local = threading.local() def __str__(self): return "JobServerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def user(self): return self._thread_local.user def setuser(self, user): if hasattr(user, "username"): self._thread_local.user = user.username else: self._thread_local.user = user def get_status(self, **kwargs): return self._root.get("healthz", params=kwargs, headers={"Accept": _TEXT_CONTENT_TYPE}) def submit_job(self, appName, classPath, data, context=None, sync=False): params = {"appName": appName, "classPath": classPath, "sync": sync} if context: params["context"] = context return self._root.post("jobs" % params, params=params, data=data, contenttype=_BINARY_CONTENT_TYPE) def job(self, job_id): return self._root.get("jobs/%s" % job_id, headers={"Accept": _JSON_CONTENT_TYPE}) def jobs(self, **kwargs): return self._root.get("jobs", params=kwargs, headers={"Accept": _JSON_CONTENT_TYPE}) def create_context(self, name, **kwargs): return self._root.post("contexts/%s" % name, params=kwargs, contenttype=_BINARY_CONTENT_TYPE) def contexts(self, **kwargs): return self._root.get("contexts", params=kwargs, headers={"Accept": _JSON_CONTENT_TYPE}) def delete_context(self, name, **kwargs): return self._root.delete("contexts/%s" % name) def upload_jar(self, app_name, data): return self._root.post("jars/%s" % app_name, data=data, contenttype=_BINARY_CONTENT_TYPE) def jars(self, **kwargs): return self._root.get("jars", params=kwargs, headers={"Accept": _JSON_CONTENT_TYPE})
class LivyClient(object): def __init__(self, livy_url): self._url = posixpath.join(livy_url) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = SECURITY_ENABLED.get() self._csrf_enabled = CSRF_ENABLED.get() self._thread_local = threading.local() if self.security_enabled: self._client.set_kerberos_auth() if self.csrf_enabled: self._client.set_headers({'X-Requested-By': 'hue'}) self._client.set_verify(SSL_CERT_CA_VERIFY.get()) def __str__(self): return "LivyClient at %s" % (self._url, ) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def csrf_enabled(self): return self._csrf_enabled @property def user(self): return self._thread_local.user def setuser(self, user): if hasattr(user, 'username'): self._thread_local.user = user.username else: self._thread_local.user = user def get_status(self): return self._root.get('sessions') def get_log(self, uuid, startFrom=None, size=None): params = {} if startFrom is not None: params['from'] = startFrom if size is not None: params['size'] = size response = self._root.get('sessions/%s/log' % uuid, params=params) return '\n'.join(response['log']) def create_session(self, **properties): properties['proxyUser'] = self.user.split('@')[0] if has_connectors(): # Only SQL supported via connectors currently properties['kind'] = 'sql' return self._root.post('sessions', data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE) def get_sessions(self): return self._root.get('sessions') def get_session(self, uuid): return self._root.get('sessions/%s' % uuid) def get_statements(self, uuid): return self._root.get('sessions/%s/statements' % uuid) def submit_statement(self, uuid, statement): data = {'code': statement} return self._root.post('sessions/%s/statements' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def inspect(self, uuid, statement): data = {'code': statement} return self._root.post('sessions/%s/inspect' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def fetch_data(self, session, statement): return self._root.get('sessions/%s/statements/%s' % (session, statement)) def cancel(self, session): return self._root.post('sessions/%s/interrupt' % session) def close(self, uuid): return self._root.delete('sessions/%s' % uuid) def get_batches(self): return self._root.get('batches') def submit_batch(self, properties): properties['proxyUser'] = self.user return self._root.post('batches', data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE) def get_batch(self, uuid): return self._root.get('batches/%s' % uuid) def get_batch_status(self, uuid): response = self._root.get('batches/%s/state' % uuid) return response['state'] def get_batch_log(self, uuid, startFrom=None, size=None): params = {} if startFrom is not None: params['from'] = startFrom if size is not None: params['size'] = size response = self._root.get('batches/%s/log' % uuid, params=params) return '\n'.join(response['log']) def close_batch(self, uuid): return self._root.delete('batches/%s' % uuid)
class JobServerApi(object): def __init__(self, livy_url): self._url = posixpath.join(livy_url) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = SECURITY_ENABLED.get() self._thread_local = threading.local() if self.security_enabled: self._client.set_kerberos_auth() def __str__(self): return "JobServerApi at %s" % (self._url, ) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def user(self): return self._thread_local.user def setuser(self, user): if hasattr(user, 'username'): self._thread_local.user = user.username else: self._thread_local.user = user def get_status(self): return self._root.get('sessions') def get_log(self, uuid, startFrom=None, size=None): params = {} if startFrom is not None: params['from'] = startFrom if size is not None: params['size'] = size response = self._root.get('sessions/%s/log' % uuid, params=params) return '\n'.join(response['log']) def create_session(self, **properties): properties['proxyUser'] = self.user return self._root.post('sessions', data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE) def get_sessions(self): return self._root.get('sessions') def get_session(self, uuid): return self._root.get('sessions/%s' % uuid) def get_statements(self, uuid): return self._root.get('sessions/%s/statements' % uuid) def submit_statement(self, uuid, statement): data = {'code': statement} return self._root.post('sessions/%s/statements' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def inspect(self, uuid, statement): data = {'code': statement} return self._root.post('sessions/%s/inspect' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def fetch_data(self, session, statement): return self._root.get('sessions/%s/statements/%s' % (session, statement)) def cancel(self, session): return self._root.post('sessions/%s/interrupt' % session) def close(self, uuid): return self._root.delete('sessions/%s' % uuid) def get_batches(self): return self._root.get('batches') def submit_batch(self, properties): properties['proxyUser'] = self.user return self._root.post('batches', data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE) def get_batch(self, uuid): return self._root.get('batches/%s' % uuid) def get_batch_status(self, uuid): response = self._root.get('batches/%s/state' % uuid) return response['state'] def get_batch_log(self, uuid, startFrom=None, size=None): params = {} if startFrom is not None: params['from'] = startFrom if size is not None: params['size'] = size response = self._root.get('batches/%s/log' % uuid, params=params) return '\n'.join(response['log']) def close_batch(self, uuid): return self._root.delete('batches/%s' % uuid)
class JobServerApi(object): def __init__(self, livy_url): self._url = posixpath.join(livy_url) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = SECURITY_ENABLED.get() self._csrf_enabled = CSRF_ENABLED.get() self._thread_local = threading.local() if self.security_enabled: self._client.set_kerberos_auth() if self.csrf_enabled: self._client.set_headers({'X-Requested-By' : 'hue'}) self._client.set_verify(SSL_CERT_CA_VERIFY.get()) def __str__(self): return "JobServerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def csrf_enabled(self): return self._csrf_enabled @property def user(self): return self._thread_local.user def setuser(self, user): if hasattr(user, 'username'): self._thread_local.user = user.username else: self._thread_local.user = user def get_status(self): return self._root.get('sessions') def get_log(self, uuid, startFrom=None, size=None): params = {} if startFrom is not None: params['from'] = startFrom if size is not None: params['size'] = size response = self._root.get('sessions/%s/log' % uuid, params=params) return '\n'.join(response['log']) def create_session(self, **properties): properties['proxyUser'] = self.user return self._root.post('sessions', data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE) def get_sessions(self): return self._root.get('sessions') def get_session(self, uuid): return self._root.get('sessions/%s' % uuid) def get_statements(self, uuid): return self._root.get('sessions/%s/statements' % uuid) def submit_statement(self, uuid, statement): data = {'code': statement} return self._root.post('sessions/%s/statements' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def inspect(self, uuid, statement): data = {'code': statement} return self._root.post('sessions/%s/inspect' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def fetch_data(self, session, statement): return self._root.get('sessions/%s/statements/%s' % (session, statement)) def cancel(self, session): return self._root.post('sessions/%s/interrupt' % session) def close(self, uuid): return self._root.delete('sessions/%s' % uuid) def get_batches(self): return self._root.get('batches') def submit_batch(self, properties): properties['proxyUser'] = self.user return self._root.post('batches', data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE) def get_batch(self, uuid): return self._root.get('batches/%s' % uuid) def get_batch_status(self, uuid): response = self._root.get('batches/%s/state' % uuid) return response['state'] def get_batch_log(self, uuid, startFrom=None, size=None): params = {} if startFrom is not None: params['from'] = startFrom if size is not None: params['size'] = size response = self._root.get('batches/%s/log' % uuid, params=params) return '\n'.join(response['log']) def close_batch(self, uuid): return self._root.delete('batches/%s' % uuid)
class FlinkSqlClient(): ''' Implements https://github.com/ververica/flink-sql-gateway Could be a pip module or sqlalchemy dialect in the future. ''' def __init__(self, user, api_url): self.user = user self._url = posixpath.join(api_url + '/' + _API_VERSION + '/') self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) def __str__(self): return "FlinkClient at %s" % (self._url, ) def info(self): return self._root.get('info') def create_session(self, **properties): data = { "session_name": "test", # optional "planner": "blink", # required, "old"/"blink" "execution_type": "streaming", # required, "batch"/"streaming" "properties": { # optional "key": "value" } } data.update(properties) return self._root.post('sessions', data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def session_heartbeat(self, session_id): return self._root.post('sessions/%(session_id)s/heartbeat' % {'session_id': session_id}) def execute_statement(self, session_id, statement): data = { "statement": statement, # required "execution_timeout": "" # execution time limit in milliseconds, optional, but required for stream SELECT ? } return self._root.post('sessions/%(session_id)s/statements' % {'session_id': session_id}, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def fetch_status(self, session_id, job_id): return self._root.get( 'sessions/%(session_id)s/jobs/%(job_id)s/status' % { 'session_id': session_id, 'job_id': job_id }) def fetch_results(self, session_id, job_id, token=0): return self._root.get( 'sessions/%(session_id)s/jobs/%(job_id)s/result/%(token)s' % { 'session_id': session_id, 'job_id': job_id, 'token': token }) def close_statement(self, session_id, job_id): return self._root.delete('sessions/%(session_id)s/jobs/%(job_id)s' % { 'session_id': session_id, 'job_id': job_id, }) def close_session(self, session_id): return self._root.delete('sessions/%(session_id)s' % { 'session_id': session_id, })
class JobServerApi(object): def __init__(self, oozie_url): self._url = posixpath.join(oozie_url) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = False self._thread_local = threading.local() def __str__(self): return "JobServerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def user(self): return self._thread_local.user def setuser(self, user): if hasattr(user, "username"): self._thread_local.user = user.username else: self._thread_local.user = user def get_status(self): return self._root.get("sessions") def get_log(self, uuid, startFrom=None, size=None): params = {} if startFrom is not None: params["from"] = startFrom if size is not None: params["size"] = size response = self._root.get("sessions/%s/log" % uuid, params=params) return "\n".join(response["log"]) def create_session(self, **properties): properties["proxyUser"] = self.user return self._root.post("sessions", data=json.dumps(properties), contenttype="application/json") def get_session(self, uuid): return self._root.get("sessions/%s" % uuid) def submit_statement(self, uuid, statement): data = {"code": statement} return self._root.post("sessions/%s/statements" % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def inspect(self, uuid, statement): data = {"code": statement} return self._root.post("sessions/%s/inspect" % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def fetch_data(self, session, statement): return self._root.get("sessions/%s/statements/%s" % (session, statement)) def cancel(self, session): return self._root.post("sessions/%s/interrupt" % session) def close(self, uuid): return self._root.delete("sessions/%s" % uuid) def get_batches(self): return self._root.get("batches") def submit_batch(self, properties): properties["proxyUser"] = self.user return self._root.post("batches", data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE) def get_batch(self, uuid): return self._root.get("batches/%s" % uuid) def get_batch_status(self, uuid): response = self._root.get("batches/%s/state" % uuid) return response["state"] def get_batch_log(self, uuid, startFrom=None, size=None): params = {} if startFrom is not None: params["from"] = startFrom if size is not None: params["size"] = size response = self._root.get("batches/%s/log" % uuid, params=params) return "\n".join(response["log"]) def close_batch(self, uuid): return self._root.delete("batches/%s" % uuid)
class JobServerApi(object): def __init__(self, oozie_url): self._url = posixpath.join(oozie_url) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = False # To store user info self._thread_local = threading.local() def __str__(self): return "JobServerApi at %s" % (self._url, ) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def user(self): return self._thread_local.user def setuser(self, user): if hasattr(user, 'username'): self._thread_local.user = user.username else: self._thread_local.user = user def get_status(self, **kwargs): return self._root.get('healthz', params=kwargs, headers={'Accept': _TEXT_CONTENT_TYPE}) def submit_job(self, appName, classPath, data, context=None, sync=False): params = {'appName': appName, 'classPath': classPath, 'sync': sync} if context: params['context'] = context return self._root.post('jobs' % params, params=params, data=data, contenttype=_BINARY_CONTENT_TYPE) def job(self, job_id): return self._root.get('jobs/%s' % job_id, headers={'Accept': _JSON_CONTENT_TYPE}) def jobs(self, **kwargs): return self._root.get('jobs', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def create_context(self, name, **kwargs): return self._root.post('contexts/%s' % name, params=kwargs, contenttype=_BINARY_CONTENT_TYPE) def contexts(self, **kwargs): return self._root.get('contexts', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def delete_context(self, name, **kwargs): return self._root.delete('contexts/%s' % name) def upload_jar(self, app_name, data): return self._root.post('jars/%s' % app_name, data=data, contenttype=_BINARY_CONTENT_TYPE) def jars(self, **kwargs): return self._root.get('jars', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
class JobServerApi(object): def __init__(self, oozie_url): self._url = posixpath.join(oozie_url) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = False self._thread_local = threading.local() def __str__(self): return "JobServerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def user(self): return self._thread_local.user def setuser(self, user): if hasattr(user, 'username'): self._thread_local.user = user.username else: self._thread_local.user = user def get_status(self): return self._root.get('sessions') def create_session(self, **kwargs): return self._root.post('sessions', data=json.dumps(kwargs), contenttype='application/json') def get_session(self, uuid): return self._root.get('sessions/%s' % uuid) def submit_statement(self, uuid, statement): data = {'code': statement} return self._root.post('sessions/%s/statements' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def inspect(self, uuid, statement): data = {'code': statement} return self._root.post('sessions/%s/inspect' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def fetch_data(self, session, statement): return self._root.get('sessions/%s/statements/%s' % (session, statement)) def cancel(self, session): return self._root.post('sessions/%s/interrupt' % session) def get_batches(self): return self._root.get('batches') def submit_batch(self, properties): return self._root.post('batches', data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE) def get_batch(self, uuid): return self._root.get('batches/%s' % uuid) def get_batch_status(self, uuid): response = self._root.get('batches/%s/state' % uuid) return response['state'] def get_batch_log(self, uuid, startFrom=None, size=None): params = {} if startFrom is not None: params['from'] = startFrom if size is not None: params['size'] = size response = self._root.get('batches/%s/log' % uuid, params=params) return '\n'.join(response['log']) def delete_batch(self, uuid): return self._root.delete('batches/%s' % uuid)
class JobServerApi(object): def __init__(self, oozie_url): self._url = posixpath.join(oozie_url) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = False self._thread_local = threading.local() def __str__(self): return "JobServerApi at %s" % (self._url, ) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def user(self): return self._thread_local.user def setuser(self, user): if hasattr(user, 'username'): self._thread_local.user = user.username else: self._thread_local.user = user def get_status(self): return self._root.get('sessions') def create_session(self, **kwargs): return self._root.post('sessions', data=json.dumps(kwargs), contenttype='application/json') def get_session(self, uuid): return self._root.get('sessions/%s' % uuid) def submit_statement(self, uuid, statement): data = {'code': statement} return self._root.post('sessions/%s/statements' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def inspect(self, uuid, statement): data = {'code': statement} return self._root.post('sessions/%s/inspect' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE) def fetch_data(self, session, statement): return self._root.get('sessions/%s/statements/%s' % (session, statement)) def cancel(self, session): return self._root.post('sessions/%s/interrupt' % session) def get_batches(self): return self._root.get('batches') def submit_batch(self, properties): return self._root.post('batches', data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE) def get_batch(self, uuid): return self._root.get('batches/%s' % uuid) def delete_batch(self, uuid): return self._root.delete('batches/%s' % uuid)