class JobServerApi(object):
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = False
# To store user info
self._thread_local = threading.local()
def __str__(self):
return "JobServerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
return self._thread_local.user
def setuser(self, user):
if hasattr(user, "username"):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def get_status(self, **kwargs):
return self._root.get("healthz", params=kwargs, headers={"Accept": _TEXT_CONTENT_TYPE})
def submit_job(self, appName, classPath, data, context=None, sync=False):
params = {"appName": appName, "classPath": classPath, "sync": sync}
if context:
params["context"] = context
return self._root.post("jobs" % params, params=params, data=data, contenttype=_BINARY_CONTENT_TYPE)
def job(self, job_id):
return self._root.get("jobs/%s" % job_id, headers={"Accept": _JSON_CONTENT_TYPE})
def jobs(self, **kwargs):
return self._root.get("jobs", params=kwargs, headers={"Accept": _JSON_CONTENT_TYPE})
def create_context(self, name, **kwargs):
return self._root.post("contexts/%s" % name, params=kwargs, contenttype=_BINARY_CONTENT_TYPE)
def contexts(self, **kwargs):
return self._root.get("contexts", params=kwargs, headers={"Accept": _JSON_CONTENT_TYPE})
def delete_context(self, name, **kwargs):
return self._root.delete("contexts/%s" % name)
def upload_jar(self, app_name, data):
return self._root.post("jars/%s" % app_name, data=data, contenttype=_BINARY_CONTENT_TYPE)
def jars(self, **kwargs):
return self._root.get("jars", params=kwargs, headers={"Accept": _JSON_CONTENT_TYPE})
class LivyClient(object):
def __init__(self, livy_url):
self._url = posixpath.join(livy_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = SECURITY_ENABLED.get()
self._csrf_enabled = CSRF_ENABLED.get()
self._thread_local = threading.local()
if self.security_enabled:
self._client.set_kerberos_auth()
if self.csrf_enabled:
self._client.set_headers({'X-Requested-By': 'hue'})
self._client.set_verify(SSL_CERT_CA_VERIFY.get())
def __str__(self):
return "LivyClient at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def csrf_enabled(self):
return self._csrf_enabled
@property
def user(self):
return self._thread_local.user
def setuser(self, user):
if hasattr(user, 'username'):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def get_status(self):
return self._root.get('sessions')
def get_log(self, uuid, startFrom=None, size=None):
params = {}
if startFrom is not None:
params['from'] = startFrom
if size is not None:
params['size'] = size
response = self._root.get('sessions/%s/log' % uuid, params=params)
return '\n'.join(response['log'])
def create_session(self, **properties):
properties['proxyUser'] = self.user.split('@')[0]
if has_connectors(): # Only SQL supported via connectors currently
properties['kind'] = 'sql'
return self._root.post('sessions',
data=json.dumps(properties),
contenttype=_JSON_CONTENT_TYPE)
def get_sessions(self):
return self._root.get('sessions')
def get_session(self, uuid):
return self._root.get('sessions/%s' % uuid)
def get_statements(self, uuid):
return self._root.get('sessions/%s/statements' % uuid)
def submit_statement(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/statements' % uuid,
data=json.dumps(data),
contenttype=_JSON_CONTENT_TYPE)
def inspect(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/inspect' % uuid,
data=json.dumps(data),
contenttype=_JSON_CONTENT_TYPE)
def fetch_data(self, session, statement):
return self._root.get('sessions/%s/statements/%s' %
(session, statement))
def cancel(self, session):
return self._root.post('sessions/%s/interrupt' % session)
def close(self, uuid):
return self._root.delete('sessions/%s' % uuid)
def get_batches(self):
return self._root.get('batches')
def submit_batch(self, properties):
properties['proxyUser'] = self.user
return self._root.post('batches',
data=json.dumps(properties),
contenttype=_JSON_CONTENT_TYPE)
def get_batch(self, uuid):
return self._root.get('batches/%s' % uuid)
def get_batch_status(self, uuid):
response = self._root.get('batches/%s/state' % uuid)
return response['state']
def get_batch_log(self, uuid, startFrom=None, size=None):
params = {}
if startFrom is not None:
params['from'] = startFrom
if size is not None:
params['size'] = size
response = self._root.get('batches/%s/log' % uuid, params=params)
return '\n'.join(response['log'])
def close_batch(self, uuid):
return self._root.delete('batches/%s' % uuid)
class JobServerApi(object):
def __init__(self, livy_url):
self._url = posixpath.join(livy_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = SECURITY_ENABLED.get()
self._thread_local = threading.local()
if self.security_enabled:
self._client.set_kerberos_auth()
def __str__(self):
return "JobServerApi at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
return self._thread_local.user
def setuser(self, user):
if hasattr(user, 'username'):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def get_status(self):
return self._root.get('sessions')
def get_log(self, uuid, startFrom=None, size=None):
params = {}
if startFrom is not None:
params['from'] = startFrom
if size is not None:
params['size'] = size
response = self._root.get('sessions/%s/log' % uuid, params=params)
return '\n'.join(response['log'])
def create_session(self, **properties):
properties['proxyUser'] = self.user
return self._root.post('sessions',
data=json.dumps(properties),
contenttype=_JSON_CONTENT_TYPE)
def get_sessions(self):
return self._root.get('sessions')
def get_session(self, uuid):
return self._root.get('sessions/%s' % uuid)
def get_statements(self, uuid):
return self._root.get('sessions/%s/statements' % uuid)
def submit_statement(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/statements' % uuid,
data=json.dumps(data),
contenttype=_JSON_CONTENT_TYPE)
def inspect(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/inspect' % uuid,
data=json.dumps(data),
contenttype=_JSON_CONTENT_TYPE)
def fetch_data(self, session, statement):
return self._root.get('sessions/%s/statements/%s' %
(session, statement))
def cancel(self, session):
return self._root.post('sessions/%s/interrupt' % session)
def close(self, uuid):
return self._root.delete('sessions/%s' % uuid)
def get_batches(self):
return self._root.get('batches')
def submit_batch(self, properties):
properties['proxyUser'] = self.user
return self._root.post('batches',
data=json.dumps(properties),
contenttype=_JSON_CONTENT_TYPE)
def get_batch(self, uuid):
return self._root.get('batches/%s' % uuid)
def get_batch_status(self, uuid):
response = self._root.get('batches/%s/state' % uuid)
return response['state']
def get_batch_log(self, uuid, startFrom=None, size=None):
params = {}
if startFrom is not None:
params['from'] = startFrom
if size is not None:
params['size'] = size
response = self._root.get('batches/%s/log' % uuid, params=params)
return '\n'.join(response['log'])
def close_batch(self, uuid):
return self._root.delete('batches/%s' % uuid)
class JobServerApi(object):
def __init__(self, livy_url):
self._url = posixpath.join(livy_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = SECURITY_ENABLED.get()
self._csrf_enabled = CSRF_ENABLED.get()
self._thread_local = threading.local()
if self.security_enabled:
self._client.set_kerberos_auth()
if self.csrf_enabled:
self._client.set_headers({'X-Requested-By' : 'hue'})
self._client.set_verify(SSL_CERT_CA_VERIFY.get())
def __str__(self):
return "JobServerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def csrf_enabled(self):
return self._csrf_enabled
@property
def user(self):
return self._thread_local.user
def setuser(self, user):
if hasattr(user, 'username'):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def get_status(self):
return self._root.get('sessions')
def get_log(self, uuid, startFrom=None, size=None):
params = {}
if startFrom is not None:
params['from'] = startFrom
if size is not None:
params['size'] = size
response = self._root.get('sessions/%s/log' % uuid, params=params)
return '\n'.join(response['log'])
def create_session(self, **properties):
properties['proxyUser'] = self.user
return self._root.post('sessions', data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE)
def get_sessions(self):
return self._root.get('sessions')
def get_session(self, uuid):
return self._root.get('sessions/%s' % uuid)
def get_statements(self, uuid):
return self._root.get('sessions/%s/statements' % uuid)
def submit_statement(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/statements' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
def inspect(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/inspect' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
def fetch_data(self, session, statement):
return self._root.get('sessions/%s/statements/%s' % (session, statement))
def cancel(self, session):
return self._root.post('sessions/%s/interrupt' % session)
def close(self, uuid):
return self._root.delete('sessions/%s' % uuid)
def get_batches(self):
return self._root.get('batches')
def submit_batch(self, properties):
properties['proxyUser'] = self.user
return self._root.post('batches', data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE)
def get_batch(self, uuid):
return self._root.get('batches/%s' % uuid)
def get_batch_status(self, uuid):
response = self._root.get('batches/%s/state' % uuid)
return response['state']
def get_batch_log(self, uuid, startFrom=None, size=None):
params = {}
if startFrom is not None:
params['from'] = startFrom
if size is not None:
params['size'] = size
response = self._root.get('batches/%s/log' % uuid, params=params)
return '\n'.join(response['log'])
def close_batch(self, uuid):
return self._root.delete('batches/%s' % uuid)
class FlinkSqlClient():
'''
Implements https://github.com/ververica/flink-sql-gateway
Could be a pip module or sqlalchemy dialect in the future.
'''
def __init__(self, user, api_url):
self.user = user
self._url = posixpath.join(api_url + '/' + _API_VERSION + '/')
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
def __str__(self):
return "FlinkClient at %s" % (self._url, )
def info(self):
return self._root.get('info')
def create_session(self, **properties):
data = {
"session_name": "test", # optional
"planner": "blink", # required, "old"/"blink"
"execution_type": "streaming", # required, "batch"/"streaming"
"properties": { # optional
"key": "value"
}
}
data.update(properties)
return self._root.post('sessions',
data=json.dumps(data),
contenttype=_JSON_CONTENT_TYPE)
def session_heartbeat(self, session_id):
return self._root.post('sessions/%(session_id)s/heartbeat' %
{'session_id': session_id})
def execute_statement(self, session_id, statement):
data = {
"statement": statement, # required
"execution_timeout":
"" # execution time limit in milliseconds, optional, but required for stream SELECT ?
}
return self._root.post('sessions/%(session_id)s/statements' %
{'session_id': session_id},
data=json.dumps(data),
contenttype=_JSON_CONTENT_TYPE)
def fetch_status(self, session_id, job_id):
return self._root.get(
'sessions/%(session_id)s/jobs/%(job_id)s/status' % {
'session_id': session_id,
'job_id': job_id
})
def fetch_results(self, session_id, job_id, token=0):
return self._root.get(
'sessions/%(session_id)s/jobs/%(job_id)s/result/%(token)s' % {
'session_id': session_id,
'job_id': job_id,
'token': token
})
def close_statement(self, session_id, job_id):
return self._root.delete('sessions/%(session_id)s/jobs/%(job_id)s' % {
'session_id': session_id,
'job_id': job_id,
})
def close_session(self, session_id):
return self._root.delete('sessions/%(session_id)s' % {
'session_id': session_id,
})
class JobServerApi(object):
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
def __str__(self):
return "JobServerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
return self._thread_local.user
def setuser(self, user):
if hasattr(user, "username"):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def get_status(self):
return self._root.get("sessions")
def get_log(self, uuid, startFrom=None, size=None):
params = {}
if startFrom is not None:
params["from"] = startFrom
if size is not None:
params["size"] = size
response = self._root.get("sessions/%s/log" % uuid, params=params)
return "\n".join(response["log"])
def create_session(self, **properties):
properties["proxyUser"] = self.user
return self._root.post("sessions", data=json.dumps(properties), contenttype="application/json")
def get_session(self, uuid):
return self._root.get("sessions/%s" % uuid)
def submit_statement(self, uuid, statement):
data = {"code": statement}
return self._root.post("sessions/%s/statements" % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
def inspect(self, uuid, statement):
data = {"code": statement}
return self._root.post("sessions/%s/inspect" % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
def fetch_data(self, session, statement):
return self._root.get("sessions/%s/statements/%s" % (session, statement))
def cancel(self, session):
return self._root.post("sessions/%s/interrupt" % session)
def close(self, uuid):
return self._root.delete("sessions/%s" % uuid)
def get_batches(self):
return self._root.get("batches")
def submit_batch(self, properties):
properties["proxyUser"] = self.user
return self._root.post("batches", data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE)
def get_batch(self, uuid):
return self._root.get("batches/%s" % uuid)
def get_batch_status(self, uuid):
response = self._root.get("batches/%s/state" % uuid)
return response["state"]
def get_batch_log(self, uuid, startFrom=None, size=None):
params = {}
if startFrom is not None:
params["from"] = startFrom
if size is not None:
params["size"] = size
response = self._root.get("batches/%s/log" % uuid, params=params)
return "\n".join(response["log"])
def close_batch(self, uuid):
return self._root.delete("batches/%s" % uuid)
class JobServerApi(object):
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = False
# To store user info
self._thread_local = threading.local()
def __str__(self):
return "JobServerApi at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
return self._thread_local.user
def setuser(self, user):
if hasattr(user, 'username'):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def get_status(self, **kwargs):
return self._root.get('healthz',
params=kwargs,
headers={'Accept': _TEXT_CONTENT_TYPE})
def submit_job(self, appName, classPath, data, context=None, sync=False):
params = {'appName': appName, 'classPath': classPath, 'sync': sync}
if context:
params['context'] = context
return self._root.post('jobs' % params,
params=params,
data=data,
contenttype=_BINARY_CONTENT_TYPE)
def job(self, job_id):
return self._root.get('jobs/%s' % job_id,
headers={'Accept': _JSON_CONTENT_TYPE})
def jobs(self, **kwargs):
return self._root.get('jobs',
params=kwargs,
headers={'Accept': _JSON_CONTENT_TYPE})
def create_context(self, name, **kwargs):
return self._root.post('contexts/%s' % name,
params=kwargs,
contenttype=_BINARY_CONTENT_TYPE)
def contexts(self, **kwargs):
return self._root.get('contexts',
params=kwargs,
headers={'Accept': _JSON_CONTENT_TYPE})
def delete_context(self, name, **kwargs):
return self._root.delete('contexts/%s' % name)
def upload_jar(self, app_name, data):
return self._root.post('jars/%s' % app_name,
data=data,
contenttype=_BINARY_CONTENT_TYPE)
def jars(self, **kwargs):
return self._root.get('jars',
params=kwargs,
headers={'Accept': _JSON_CONTENT_TYPE})
class JobServerApi(object):
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
def __str__(self):
return "JobServerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
return self._thread_local.user
def setuser(self, user):
if hasattr(user, 'username'):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def get_status(self):
return self._root.get('sessions')
def create_session(self, **kwargs):
return self._root.post('sessions', data=json.dumps(kwargs), contenttype='application/json')
def get_session(self, uuid):
return self._root.get('sessions/%s' % uuid)
def submit_statement(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/statements' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
def inspect(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/inspect' % uuid, data=json.dumps(data), contenttype=_JSON_CONTENT_TYPE)
def fetch_data(self, session, statement):
return self._root.get('sessions/%s/statements/%s' % (session, statement))
def cancel(self, session):
return self._root.post('sessions/%s/interrupt' % session)
def get_batches(self):
return self._root.get('batches')
def submit_batch(self, properties):
return self._root.post('batches', data=json.dumps(properties), contenttype=_JSON_CONTENT_TYPE)
def get_batch(self, uuid):
return self._root.get('batches/%s' % uuid)
def get_batch_status(self, uuid):
response = self._root.get('batches/%s/state' % uuid)
return response['state']
def get_batch_log(self, uuid, startFrom=None, size=None):
params = {}
if startFrom is not None:
params['from'] = startFrom
if size is not None:
params['size'] = size
response = self._root.get('batches/%s/log' % uuid, params=params)
return '\n'.join(response['log'])
def delete_batch(self, uuid):
return self._root.delete('batches/%s' % uuid)
class JobServerApi(object):
def __init__(self, oozie_url):
self._url = posixpath.join(oozie_url)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = False
self._thread_local = threading.local()
def __str__(self):
return "JobServerApi at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
return self._thread_local.user
def setuser(self, user):
if hasattr(user, 'username'):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def get_status(self):
return self._root.get('sessions')
def create_session(self, **kwargs):
return self._root.post('sessions',
data=json.dumps(kwargs),
contenttype='application/json')
def get_session(self, uuid):
return self._root.get('sessions/%s' % uuid)
def submit_statement(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/statements' % uuid,
data=json.dumps(data),
contenttype=_JSON_CONTENT_TYPE)
def inspect(self, uuid, statement):
data = {'code': statement}
return self._root.post('sessions/%s/inspect' % uuid,
data=json.dumps(data),
contenttype=_JSON_CONTENT_TYPE)
def fetch_data(self, session, statement):
return self._root.get('sessions/%s/statements/%s' %
(session, statement))
def cancel(self, session):
return self._root.post('sessions/%s/interrupt' % session)
def get_batches(self):
return self._root.get('batches')
def submit_batch(self, properties):
return self._root.post('batches',
data=json.dumps(properties),
contenttype=_JSON_CONTENT_TYPE)
def get_batch(self, uuid):
return self._root.get('batches/%s' % uuid)
def delete_batch(self, uuid):
return self._root.delete('batches/%s' % uuid)
