class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "ResourceManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def cluster(self, **kwargs):
return self._root.get('cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def apps(self, **kwargs):
return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def app(self, app_id):
return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def kill(self, app_id):
return self._root.put('cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE)
class ResourceManagerApi(object):
def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
self._ssl_cert_ca_verify = ssl_cert_ca_verify
if self._security_enabled:
self._client.set_kerberos_auth()
if ssl_cert_ca_verify:
self._client.set_verify(True)
def __str__(self):
return "ResourceManagerApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def cluster(self, **kwargs):
return self._root.get('cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def apps(self, **kwargs):
return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE})
def app(self, app_id):
return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE})
def kill(self, app_id):
return self._root.put('cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE)
class OozieApi(object):
def __init__(self,
oozie_url,
user,
security_enabled=False,
api_version=API_VERSION,
ssl_cert_ca_verify=True):
self._url = posixpath.join(oozie_url, api_version)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
if hasattr(user, 'username'):
self.user = user.username
else:
self.user = user
self.api_version = api_version
def __str__(self):
return "OozieApi at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def _get_params(self):
if self.security_enabled:
return {'doAs': self.user, 'timezone': TIME_ZONE.get()}
return {
'user.name': DEFAULT_USER,
'doAs': self.user,
'timezone': TIME_ZONE.get()
}
def _get_oozie_properties(self, properties=None):
defaults = {
'user.name': self.user,
}
if properties is not None:
defaults.update(properties)
return defaults
VALID_JOB_FILTERS = ('name', 'user', 'group', 'status', 'startcreatedtime',
'text')
VALID_LOG_FILTERS = set(('recent', 'limit', 'loglevel', 'text'))
def get_jobs(self, jobtype, offset=None, cnt=None, filters=None):
"""
Get a list of Oozie jobs.
Note that offset is 1-based.
kwargs is used for filtering and may be one of VALID_FILTERS: name, user, group, status
"""
params = self._get_params()
if offset is not None:
params['offset'] = str(offset)
if cnt is not None:
params['len'] = str(cnt)
if filters is None:
filters = []
params['jobtype'] = jobtype
filter_list = []
for key, val in filters:
if key not in OozieApi.VALID_JOB_FILTERS:
raise ValueError(
'"%s" is not a valid filter for selecting jobs' % (key, ))
filter_list.append('%s=%s' % (key, val))
params['filter'] = ';'.join(filter_list)
# Send the request
resp = self._root.get('jobs', params)
if jobtype == 'wf':
wf_list = WorkflowList(self, resp, filters=filters)
elif jobtype == 'coord':
wf_list = CoordinatorList(self, resp, filters=filters)
else:
wf_list = BundleList(self, resp, filters=filters)
return wf_list
def get_workflows(self, offset=None, cnt=None, filters=None):
return self.get_jobs('wf', offset, cnt, filters)
def get_coordinators(self, offset=None, cnt=None, filters=None):
return self.get_jobs('coord', offset, cnt, filters)
def get_bundles(self, offset=None, cnt=None, filters=None):
return self.get_jobs('bundle', offset, cnt, filters)
# TODO: make get_job accept any jobid
def get_job(self, jobid):
"""
get_job(jobid) -> Workflow
"""
params = self._get_params()
resp = self._root.get('job/%s' % (jobid, ), params)
wf = Workflow(self, resp)
return wf
def get_coordinator(self, jobid, offset=None, cnt=None, filters=None):
params = self._get_params()
if offset is not None:
params['offset'] = str(offset)
if cnt is not None:
params['len'] = str(cnt)
if filters is None:
filters = {}
params.update({'order': 'desc'})
filter_list = []
for key, val in filters:
if key not in OozieApi.VALID_JOB_FILTERS:
raise ValueError(
'"%s" is not a valid filter for selecting jobs' % (key, ))
filter_list.append('%s=%s' % (key, val))
params['filter'] = ';'.join(filter_list)
resp = self._root.get('job/%s' % (jobid, ), params)
return Coordinator(self, resp)
def get_bundle(self, jobid):
params = self._get_params()
resp = self._root.get('job/%s' % (jobid, ), params)
return Bundle(self, resp)
def get_job_definition(self, jobid):
"""
get_job_definition(jobid) -> Definition (xml string)
"""
params = self._get_params()
params['show'] = 'definition'
return self._root.get('job/%s' % (jobid, ), params)
def get_job_log(self, jobid, logfilter=None):
"""
get_job_log(jobid) -> Log (xml string)
"""
params = self._get_params()
params['show'] = 'log'
filter_list = []
if logfilter is None:
logfilter = []
for key, val in logfilter:
if key not in OozieApi.VALID_LOG_FILTERS:
raise ValueError('"%s" is not a valid filter for job logs' %
(key, ))
filter_list.append('%s=%s' % (key, val))
params['logfilter'] = ';'.join(filter_list)
return self._root.get('job/%s' % (jobid, ), params)
def get_job_status(self, jobid):
params = self._get_params()
params['show'] = 'status'
xml = self._root.get('job/%s' % (jobid, ), params)
return xml
def get_action(self, action_id):
if 'C@' in action_id:
Klass = CoordinatorAction
elif 'B@' in action_id:
Klass = BundleAction
else:
Klass = WorkflowAction
params = self._get_params()
resp = self._root.get('job/%s' % (action_id, ), params)
return Klass(resp)
def job_control(self, jobid, action, properties=None, parameters=None):
"""
job_control(jobid, action) -> None
Raise RestException on error.
"""
if action not in ('start', 'suspend', 'resume', 'kill', 'rerun',
'coord-rerun', 'bundle-rerun', 'change', 'ignore',
'update'):
msg = 'Invalid oozie job action: %s' % (action, )
LOG.error(msg)
raise ValueError(msg)
properties = self._get_oozie_properties(properties)
params = self._get_params()
params['action'] = action
if parameters is not None:
params.update(parameters)
return self._root.put('job/%s' % jobid,
params,
data=config_gen(properties),
contenttype=_XML_CONTENT_TYPE)
def submit_workflow(self, application_path, properties=None):
"""
submit_workflow(application_path, properties=None) -> jobid
Raise RestException on error.
"""
defaults = {
'oozie.wf.application.path': application_path,
'user.name': self.user,
}
if properties is not None:
defaults.update(properties)
properties = defaults
return self.submit_job(properties)
# Is name actually submit_coord?
def submit_job(self, properties=None):
"""
submit_job(properties=None, id=None) -> jobid
Raise RestException on error.
"""
defaults = {
'user.name': self.user,
}
if properties is not None:
defaults.update(properties)
properties = defaults
params = self._get_params()
resp = self._root.post('jobs',
params,
data=config_gen(properties),
contenttype=_XML_CONTENT_TYPE)
return resp['id']
def dryrun(self, properties=None):
defaults = {
'user.name': self.user,
}
if properties is not None:
defaults.update(properties)
properties = defaults
params = self._get_params()
params['action'] = 'dryrun'
return self._root.post('jobs',
params,
data=config_gen(properties),
contenttype=_XML_CONTENT_TYPE)
def rerun(self, jobid, properties=None, params=None):
properties = self._get_oozie_properties(properties)
if params is None:
params = self._get_params()
else:
self._get_params().update(params)
params['action'] = 'rerun'
return self._root.put('job/%s' % jobid,
params,
data=config_gen(properties),
contenttype=_XML_CONTENT_TYPE)
def get_build_version(self):
"""
get_build_version() -> Build version (dictionary)
"""
params = self._get_params()
resp = self._root.get('admin/build-version', params)
return resp
def get_instrumentation(self):
params = self._get_params()
resp = self._root.get('admin/instrumentation', params)
return resp
def get_metrics(self):
params = self._get_params()
resp = self._root.get('admin/metrics', params)
return resp
def get_configuration(self):
"""
get_configuration() -> Oozie config (dictionary)
"""
params = self._get_params()
resp = self._root.get('admin/configuration', params)
return resp
def get_oozie_status(self):
"""
get_oozie_status() -> Oozie status (dictionary)
"""
params = self._get_params()
resp = self._root.get('admin/status', params)
return resp
def get_oozie_slas(self, **kwargs):
"""
filter=
app_name=my-sla-app
id=0000002-131206135002457-oozie-oozi-W
nominal_start=2013-06-18T00:01Z
nominal_end=2013-06-23T00:01Z
"""
params = self._get_params()
params['filter'] = ';'.join(
['%s=%s' % (key, val) for key, val in kwargs.iteritems()])
resp = self._root.get('sla', params)
return resp['slaSummaryList']
class OozieApi(object):
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store user info
self._thread_local = threading.local()
def __str__(self):
return "OozieApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
try:
return self._thread_local.user
except AttributeError:
return DEFAULT_USER
def setuser(self, user):
"""Return the previous user"""
prev = self.user
self._thread_local.user = user
return prev
def _get_params(self):
if self.security_enabled:
return { 'doAs': self.user, 'timezone': TIME_ZONE.get() }
return { 'user.name': DEFAULT_USER, 'doAs': self.user, 'timezone': TIME_ZONE.get() }
def _get_oozie_properties(self, properties=None):
defaults = {
'user.name': self.user,
}
if properties is not None:
defaults.update(properties)
return defaults
VALID_JOB_FILTERS = ('name', 'user', 'group', 'status')
def get_jobs(self, jobtype, offset=None, cnt=None, **kwargs):
"""
Get a list of Oozie jobs.
jobtype is 'wf', 'coord'
Note that offset is 1-based.
kwargs is used for filtering and may be one of VALID_FILTERS: name, user, group, status
"""
params = self._get_params()
if offset is not None:
params['offset'] = str(offset)
if cnt is not None:
params['len'] = str(cnt)
params['jobtype'] = jobtype
filter_list = [ ]
for key, val in kwargs.iteritems():
if key not in OozieApi.VALID_JOB_FILTERS:
raise ValueError('"%s" is not a valid filter for selecting jobs' % (key,))
filter_list.append('%s=%s' % (key, val))
params['filter'] = ';'.join(filter_list)
# Send the request
resp = self._root.get('jobs', params)
if jobtype == 'wf':
wf_list = WorkflowList(self, resp, filters=kwargs)
elif jobtype == 'coord':
wf_list = CoordinatorList(self, resp, filters=kwargs)
else:
wf_list = BundleList(self, resp, filters=kwargs)
return wf_list
def get_workflows(self, offset=None, cnt=None, **kwargs):
return self.get_jobs('wf', offset, cnt, **kwargs)
def get_coordinators(self, offset=None, cnt=None, **kwargs):
return self.get_jobs('coord', offset, cnt, **kwargs)
def get_bundles(self, offset=None, cnt=None, **kwargs):
return self.get_jobs('bundle', offset, cnt, **kwargs)
# TODO: make get_job accept any jobid
def get_job(self, jobid):
"""
get_job(jobid) -> Workflow
"""
params = self._get_params()
resp = self._root.get('job/%s' % (jobid,), params)
wf = Workflow(self, resp)
return wf
def get_coordinator(self, jobid):
params = self._get_params()
params.update({'len': -1})
resp = self._root.get('job/%s' % (jobid,), params)
return Coordinator(self, resp)
def get_bundle(self, jobid):
params = self._get_params()
resp = self._root.get('job/%s' % (jobid,), params)
return Bundle(self, resp)
def get_job_definition(self, jobid):
"""
get_job_definition(jobid) -> Definition (xml string)
"""
params = self._get_params()
params['show'] = 'definition'
xml = self._root.get('job/%s' % (jobid,), params)
return xml
def get_job_log(self, jobid):
"""
get_job_log(jobid) -> Log (xml string)
"""
params = self._get_params()
params['show'] = 'log'
xml = self._root.get('job/%s' % (jobid,), params)
return xml
def get_action(self, action_id):
if 'C@' in action_id:
Klass = CoordinatorAction
elif 'B@' in action_id:
Klass = BundleAction
else:
Klass = WorkflowAction
params = self._get_params()
resp = self._root.get('job/%s' % (action_id,), params)
return Klass(resp)
def job_control(self, jobid, action, properties=None, parameters=None):
"""
job_control(jobid, action) -> None
Raise RestException on error.
"""
if action not in ('start', 'suspend', 'resume', 'kill', 'rerun', 'coord-rerun', 'bundle-rerun'):
msg = 'Invalid oozie job action: %s' % (action,)
LOG.error(msg)
raise ValueError(msg)
properties = self._get_oozie_properties(properties)
params = self._get_params()
params['action'] = action
if parameters is not None:
params.update(parameters)
return self._root.put('job/%s' % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE)
def submit_workflow(self, application_path, properties=None):
"""
submit_workflow(application_path, properties=None) -> jobid
Raise RestException on error.
"""
defaults = {
'oozie.wf.application.path': application_path,
'user.name': self.user,
}
if properties is not None:
defaults.update(properties)
properties = defaults
return self.submit_job(properties)
# Is name actually submit_coord?
def submit_job(self, properties=None):
"""
submit_job(properties=None, id=None) -> jobid
Raise RestException on error.
"""
defaults = {
'user.name': self.user,
}
if properties is not None:
defaults.update(properties)
properties = defaults
params = self._get_params()
resp = self._root.post('jobs', params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE)
return resp['id']
def rerun(self, jobid, properties=None, params=None):
properties = self._get_oozie_properties(properties)
if params is None:
params = self._get_params()
else:
self._get_params().update(params)
params['action'] = 'rerun'
return self._root.put('job/%s' % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE)
def get_build_version(self):
"""
get_build_version() -> Build version (dictionary)
"""
params = self._get_params()
resp = self._root.get('admin/build-version', params)
return resp
def get_instrumentation(self):
params = self._get_params()
resp = self._root.get('admin/instrumentation', params)
return resp
def get_configuration(self):
"""
get_configuration() -> Oozie config (dictionary)
"""
params = self._get_params()
resp = self._root.get('admin/configuration', params)
return resp
def get_oozie_status(self):
"""
get_oozie_status() -> Oozie status (dictionary)
"""
params = self._get_params()
resp = self._root.get('admin/status', params)
return resp
class OozieApi(object):
def __init__(self, oozie_url, security_enabled=False, api_version=API_VERSION):
self._url = posixpath.join(oozie_url, api_version)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
self._thread_local = threading.local()
self.api_version = api_version
def __str__(self):
return "OozieApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
return self._thread_local.user
def setuser(self, user):
if hasattr(user, "username"):
self._thread_local.user = user.username
else:
self._thread_local.user = user
def _get_params(self):
if self.security_enabled:
return {"doAs": self.user, "timezone": TIME_ZONE.get()}
return {"user.name": DEFAULT_USER, "doAs": self.user, "timezone": TIME_ZONE.get()}
def _get_oozie_properties(self, properties=None):
defaults = {"user.name": self.user}
if properties is not None:
defaults.update(properties)
return defaults
VALID_JOB_FILTERS = ("name", "user", "group", "status")
def get_jobs(self, jobtype, offset=None, cnt=None, **kwargs):
"""
Get a list of Oozie jobs.
jobtype is 'wf', 'coord'
Note that offset is 1-based.
kwargs is used for filtering and may be one of VALID_FILTERS: name, user, group, status
"""
params = self._get_params()
if offset is not None:
params["offset"] = str(offset)
if cnt is not None:
params["len"] = str(cnt)
params["jobtype"] = jobtype
filter_list = []
for key, val in kwargs.iteritems():
if key not in OozieApi.VALID_JOB_FILTERS:
raise ValueError('"%s" is not a valid filter for selecting jobs' % (key,))
filter_list.append("%s=%s" % (key, val))
params["filter"] = ";".join(filter_list)
# Send the request
resp = self._root.get("jobs", params)
if jobtype == "wf":
wf_list = WorkflowList(self, resp, filters=kwargs)
elif jobtype == "coord":
wf_list = CoordinatorList(self, resp, filters=kwargs)
else:
wf_list = BundleList(self, resp, filters=kwargs)
return wf_list
def get_workflows(self, offset=None, cnt=None, **kwargs):
return self.get_jobs("wf", offset, cnt, **kwargs)
def get_coordinators(self, offset=None, cnt=None, **kwargs):
return self.get_jobs("coord", offset, cnt, **kwargs)
def get_bundles(self, offset=None, cnt=None, **kwargs):
return self.get_jobs("bundle", offset, cnt, **kwargs)
# TODO: make get_job accept any jobid
def get_job(self, jobid):
"""
get_job(jobid) -> Workflow
"""
params = self._get_params()
resp = self._root.get("job/%s" % (jobid,), params)
wf = Workflow(self, resp)
return wf
def get_coordinator(self, jobid):
params = self._get_params()
params.update({"len": -1})
resp = self._root.get("job/%s" % (jobid,), params)
return Coordinator(self, resp)
def get_bundle(self, jobid):
params = self._get_params()
resp = self._root.get("job/%s" % (jobid,), params)
return Bundle(self, resp)
def get_job_definition(self, jobid):
"""
get_job_definition(jobid) -> Definition (xml string)
"""
params = self._get_params()
params["show"] = "definition"
xml = self._root.get("job/%s" % (jobid,), params)
return xml
def get_job_log(self, jobid):
"""
get_job_log(jobid) -> Log (xml string)
"""
params = self._get_params()
params["show"] = "log"
xml = self._root.get("job/%s" % (jobid,), params)
return xml
def get_action(self, action_id):
if "C@" in action_id:
Klass = CoordinatorAction
elif "B@" in action_id:
Klass = BundleAction
else:
Klass = WorkflowAction
params = self._get_params()
resp = self._root.get("job/%s" % (action_id,), params)
return Klass(resp)
def job_control(self, jobid, action, properties=None, parameters=None):
"""
job_control(jobid, action) -> None
Raise RestException on error.
"""
if action not in ("start", "suspend", "resume", "kill", "rerun", "coord-rerun", "bundle-rerun"):
msg = "Invalid oozie job action: %s" % (action,)
LOG.error(msg)
raise ValueError(msg)
properties = self._get_oozie_properties(properties)
params = self._get_params()
params["action"] = action
if parameters is not None:
params.update(parameters)
return self._root.put("job/%s" % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE)
def submit_workflow(self, application_path, properties=None):
"""
submit_workflow(application_path, properties=None) -> jobid
Raise RestException on error.
"""
defaults = {"oozie.wf.application.path": application_path, "user.name": self.user}
if properties is not None:
defaults.update(properties)
properties = defaults
return self.submit_job(properties)
# Is name actually submit_coord?
def submit_job(self, properties=None):
"""
submit_job(properties=None, id=None) -> jobid
Raise RestException on error.
"""
defaults = {"user.name": self.user}
if properties is not None:
defaults.update(properties)
properties = defaults
params = self._get_params()
resp = self._root.post("jobs", params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE)
return resp["id"]
def rerun(self, jobid, properties=None, params=None):
properties = self._get_oozie_properties(properties)
if params is None:
params = self._get_params()
else:
self._get_params().update(params)
params["action"] = "rerun"
return self._root.put("job/%s" % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE)
def get_build_version(self):
"""
get_build_version() -> Build version (dictionary)
"""
params = self._get_params()
resp = self._root.get("admin/build-version", params)
return resp
def get_instrumentation(self):
params = self._get_params()
resp = self._root.get("admin/instrumentation", params)
return resp
def get_configuration(self):
"""
get_configuration() -> Oozie config (dictionary)
"""
params = self._get_params()
resp = self._root.get("admin/configuration", params)
return resp
def get_oozie_status(self):
"""
get_oozie_status() -> Oozie status (dictionary)
"""
params = self._get_params()
resp = self._root.get("admin/status", params)
return resp
def get_oozie_slas(self, **kwargs):
"""
filter=
app_name=my-sla-app
id=0000002-131206135002457-oozie-oozi-W
nominal_start=2013-06-18T00:01Z
nominal_end=2013-06-23T00:01Z
"""
params = self._get_params()
params["filter"] = ";".join(["%s=%s" % (key, val) for key, val in kwargs.iteritems()])
resp = self._root.get("sla", params)
return resp["slaSummaryList"]
class OozieApi(object):
def __init__(self, oozie_url, user, security_enabled=False, api_version=API_VERSION, ssl_cert_ca_verify=True):
self._url = posixpath.join(oozie_url, api_version)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store username info
if hasattr(user, 'username'):
self.user = user.username
else:
self.user = user
self.api_version = api_version
def __str__(self):
return "OozieApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def _get_params(self):
if self.security_enabled:
return { 'doAs': self.user, 'timezone': TIME_ZONE.get() }
return { 'user.name': DEFAULT_USER, 'doAs': self.user, 'timezone': TIME_ZONE.get() }
def _get_oozie_properties(self, properties=None):
defaults = {
'user.name': self.user,
}
if properties is not None:
defaults.update(properties)
return defaults
VALID_JOB_FILTERS = ('name', 'user', 'group', 'status', 'startcreatedtime')
VALID_LOG_FILTERS = {'recent', 'limit', 'loglevel', 'text'}
def get_jobs(self, jobtype, offset=None, cnt=None, filters=None):
"""
Get a list of Oozie jobs.
Note that offset is 1-based.
kwargs is used for filtering and may be one of VALID_FILTERS: name, user, group, status
"""
params = self._get_params()
if offset is not None:
params['offset'] = str(offset)
if cnt is not None:
params['len'] = str(cnt)
if filters is None:
filters = []
params['jobtype'] = jobtype
filter_list = []
for key, val in filters:
if key not in OozieApi.VALID_JOB_FILTERS:
raise ValueError('"%s" is not a valid filter for selecting jobs' % (key,))
filter_list.append('%s=%s' % (key, val))
params['filter'] = ';'.join(filter_list)
# Send the request
resp = self._root.get('jobs', params)
if jobtype == 'wf':
wf_list = WorkflowList(self, resp, filters=filters)
elif jobtype == 'coord':
wf_list = CoordinatorList(self, resp, filters=filters)
else:
wf_list = BundleList(self, resp, filters=filters)
return wf_list
def get_workflows(self, offset=None, cnt=None, filters=None):
return self.get_jobs('wf', offset, cnt, filters)
def get_coordinators(self, offset=None, cnt=None, filters=None):
return self.get_jobs('coord', offset, cnt, filters)
def get_bundles(self, offset=None, cnt=None, filters=None):
return self.get_jobs('bundle', offset, cnt, filters)
# TODO: make get_job accept any jobid
def get_job(self, jobid):
"""
get_job(jobid) -> Workflow
"""
params = self._get_params()
resp = self._root.get('job/%s' % (jobid,), params)
wf = Workflow(self, resp)
return wf
def get_coordinator(self, jobid, offset=None, cnt=None, filters=None):
params = self._get_params()
if offset is not None:
params['offset'] = str(offset)
if cnt is not None:
params['len'] = str(cnt)
if filters is None:
filters = {}
params.update({'order': 'desc'})
filter_list = []
for key, val in filters:
if key not in OozieApi.VALID_JOB_FILTERS:
raise ValueError('"%s" is not a valid filter for selecting jobs' % (key,))
filter_list.append('%s=%s' % (key, val))
params['filter'] = ';'.join(filter_list)
resp = self._root.get('job/%s' % (jobid,), params)
return Coordinator(self, resp)
def get_bundle(self, jobid):
params = self._get_params()
resp = self._root.get('job/%s' % (jobid,), params)
return Bundle(self, resp)
def get_job_definition(self, jobid):
"""
get_job_definition(jobid) -> Definition (xml string)
"""
params = self._get_params()
params['show'] = 'definition'
return self._root.get('job/%s' % (jobid,), params)
def get_job_log(self, jobid, logfilter=None):
"""
get_job_log(jobid) -> Log (xml string)
"""
params = self._get_params()
params['show'] = 'log'
filter_list = []
if logfilter is None:
logfilter = []
for key, val in logfilter:
if key not in OozieApi.VALID_LOG_FILTERS:
raise ValueError('"%s" is not a valid filter for job logs' % (key,))
filter_list.append('%s=%s' % (key, val))
params['logfilter'] = ';'.join(filter_list)
return self._root.get('job/%s' % (jobid,), params)
def get_job_status(self, jobid):
params = self._get_params()
params['show'] = 'status'
xml = self._root.get('job/%s' % (jobid,), params)
return xml
def get_action(self, action_id):
if 'C@' in action_id:
Klass = CoordinatorAction
elif 'B@' in action_id:
Klass = BundleAction
else:
Klass = WorkflowAction
params = self._get_params()
resp = self._root.get('job/%s' % (action_id,), params)
return Klass(resp)
def job_control(self, jobid, action, properties=None, parameters=None):
"""
job_control(jobid, action) -> None
Raise RestException on error.
"""
if action not in ('start', 'suspend', 'resume', 'kill', 'rerun', 'coord-rerun', 'bundle-rerun', 'change', 'ignore'):
msg = 'Invalid oozie job action: %s' % (action,)
LOG.error(msg)
raise ValueError(msg)
properties = self._get_oozie_properties(properties)
params = self._get_params()
params['action'] = action
if parameters is not None:
params.update(parameters)
return self._root.put('job/%s' % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE)
def submit_workflow(self, application_path, properties=None):
"""
submit_workflow(application_path, properties=None) -> jobid
Raise RestException on error.
"""
defaults = {
'oozie.wf.application.path': application_path,
'user.name': self.user,
}
if properties is not None:
defaults.update(properties)
properties = defaults
return self.submit_job(properties)
# Is name actually submit_coord?
def submit_job(self, properties=None):
"""
submit_job(properties=None, id=None) -> jobid
Raise RestException on error.
"""
defaults = {
'user.name': self.user,
}
if properties is not None:
defaults.update(properties)
properties = defaults
params = self._get_params()
resp = self._root.post('jobs', params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE)
return resp['id']
def dryrun(self, properties=None):
defaults = {
'user.name': self.user,
}
if properties is not None:
defaults.update(properties)
properties = defaults
params = self._get_params()
params['action'] = 'dryrun'
return self._root.post('jobs', params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE)
def rerun(self, jobid, properties=None, params=None):
properties = self._get_oozie_properties(properties)
if params is None:
params = self._get_params()
else:
self._get_params().update(params)
params['action'] = 'rerun'
return self._root.put('job/%s' % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE)
def get_build_version(self):
"""
get_build_version() -> Build version (dictionary)
"""
params = self._get_params()
resp = self._root.get('admin/build-version', params)
return resp
def get_instrumentation(self):
params = self._get_params()
resp = self._root.get('admin/instrumentation', params)
return resp
def get_metrics(self):
params = self._get_params()
resp = self._root.get('admin/metrics', params)
return resp
def get_configuration(self):
"""
get_configuration() -> Oozie config (dictionary)
"""
params = self._get_params()
resp = self._root.get('admin/configuration', params)
return resp
def get_oozie_status(self):
"""
get_oozie_status() -> Oozie status (dictionary)
"""
params = self._get_params()
resp = self._root.get('admin/status', params)
return resp
def get_oozie_slas(self, **kwargs):
"""
filter=
app_name=my-sla-app
id=0000002-131206135002457-oozie-oozi-W
nominal_start=2013-06-18T00:01Z
nominal_end=2013-06-23T00:01Z
"""
params = self._get_params()
params['filter'] = ';'.join(['%s=%s' % (key, val) for key, val in kwargs.iteritems()])
resp = self._root.get('sla', params)
return resp['slaSummaryList']
class OozieApi(object):
def __init__(self, oozie_url, security_enabled=False):
self._url = posixpath.join(oozie_url, API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
if security_enabled:
self._client.set_kerberos_auth()
self._root = Resource(self._client)
self._security_enabled = security_enabled
# To store user info
self._thread_local = threading.local()
def __str__(self):
return "OozieApi at %s" % (self._url,)
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
@property
def user(self):
try:
return self._thread_local.user
except AttributeError:
return DEFAULT_USER
def setuser(self, user):
"""Return the previous user"""
prev = self.user
self._thread_local.user = user
return prev
def _get_params(self):
if self.security_enabled:
return {"doAs": self.user}
return {"user.name": DEFAULT_USER, "doAs": self.user}
VALID_JOB_FILTERS = ("name", "user", "group", "status")
def get_jobs(self, offset=None, cnt=None, **kwargs):
"""
get_jobs(offset=None, cnt=None, **kwargs) -> WorkflowList
Note that offset is 1-based.
kwargs is used for filtering and may be one of VALID_FILTERS: name, user, group, status
"""
params = self._get_params()
if offset is not None:
params["offset"] = str(offset)
if cnt is not None:
params["len"] = str(cnt)
filter_list = []
for key, val in kwargs:
if key not in OozieApi.VALID_JOB_FILTERS:
raise ValueError('"%s" is not a valid filter for selecting jobs' % (key,))
filter_list.append("%s=%s" % (key, val))
params["filter"] = ";".join(filter_list)
# Send the request
resp = self._root.get("jobs", params)
wf_list = WorkflowList(self, resp, filters=kwargs)
return wf_list
def get_job(self, jobid):
"""
get_job(jobid) -> Workflow
"""
params = self._get_params()
resp = self._root.get("job/%s" % (jobid,), params)
wf = Workflow(self, resp)
return wf
def get_job_definition(self, jobid):
"""
get_job_definition(jobid) -> Definition (xml string)
"""
params = self._get_params()
params["show"] = "definition"
xml = self._root.get("job/%s" % (jobid,), params)
return xml
def get_job_log(self, jobid):
"""
get_job_log(jobid) -> Log (xml string)
"""
params = self._get_params()
params["show"] = "log"
xml = self._root.get("job/%s" % (jobid,), params)
return xml
def job_control(self, jobid, action):
"""
job_control(jobid, action) -> None
Raise RestException on error.
"""
if action not in ("start", "suspend", "resume", "kill"):
msg = "Invalid oozie job action: %s" % (action,)
LOG.error(msg)
raise ValueError(msg)
params = self._get_params()
params["action"] = action
self._root.put("job/%s" % (jobid,), params)
def submit_workflow(self, application_path, properties=None):
"""
submit_workflow(application_path, username, properties=None) -> jobid
Submit a job to Oozie. May raise PopupException.
"""
defaults = {"oozie.wf.application.path": application_path, "user.name": self.user}
if properties is not None:
defaults.update(properties)
properties = defaults
else:
properties = defaults
params = self._get_params()
resp = self._root.post("jobs", params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE)
return resp["id"]
def get_build_version(self):
"""
get_build_version() -> Build version (dictionary)
"""
params = self._get_params()
resp = self._root.get("admin/build-version", params)
return resp
def get_instrumentation(self):
"""
get_instrumentation() -> Oozie instrumentation (dictionary)
"""
params = self._get_params()
resp = self._root.get("admin/instrumentation", params)
return resp
def get_configuration(self):
"""
get_configuration() -> Oozie config (dictionary)
"""
params = self._get_params()
resp = self._root.get("admin/configuration", params)
return resp
def get_oozie_status(self):
"""
get_oozie_status() -> Oozie status (dictionary)
"""
params = self._get_params()
resp = self._root.get("admin/status", params)
return resp
class ResourceManagerApi(object):
def __init__(self,
oozie_url,
security_enabled=False,
ssl_cert_ca_verify=False):
self._url = posixpath.join(oozie_url, 'ws', _API_VERSION)
self._client = HttpClient(self._url, logger=LOG)
self._root = Resource(self._client)
self._security_enabled = security_enabled
if self._security_enabled:
self._client.set_kerberos_auth()
self._client.set_verify(ssl_cert_ca_verify)
def __str__(self):
return "ResourceManagerApi at %s" % (self._url, )
@property
def url(self):
return self._url
@property
def security_enabled(self):
return self._security_enabled
def cluster(self, **kwargs):
return self._root.get('cluster',
params=kwargs,
headers={'Accept': _JSON_CONTENT_TYPE})
return self._execute(self._root.get,
'cluster',
params=kwargs,
headers={'Accept': _JSON_CONTENT_TYPE})
def apps(self, **kwargs):
return self._root.get('cluster/apps',
params=kwargs,
headers={'Accept': _JSON_CONTENT_TYPE})
return self._execute(self._root.get,
'cluster/apps',
params=kwargs,
headers={'Accept': _JSON_CONTENT_TYPE})
def app(self, app_id):
return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id},
headers={'Accept': _JSON_CONTENT_TYPE})
return self._execute(self._root.get,
'cluster/apps/%(app_id)s' % {'app_id': app_id},
headers={'Accept': _JSON_CONTENT_TYPE})
def kill(self, app_id):
return self._root.put('cluster/apps/%(app_id)s/state' %
{'app_id': app_id},
data=json.dumps({'state': 'KILLED'}),
contenttype=_JSON_CONTENT_TYPE)
return self._execute(self._root.put,
'cluster/apps/%(app_id)s/state' %
{'app_id': app_id},
data=json.dumps({'state': 'KILLED'}),
contenttype=_JSON_CONTENT_TYPE)
def _execute(self, function, *args, **kwargs):
response = function(*args, **kwargs)
# YARN-2605: Yarn does not use proper HTTP redirects when the standby RM has
# failed back to the master RM.
if isinstance(response, str) and response.startswith(
'This is standby RM. Redirecting to the current active RM'):
raise YarnFailoverOccurred(response)
return response
