class ResourceManagerApi(object): def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False): self._url = posixpath.join(oozie_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) def __str__(self): return "ResourceManagerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def cluster(self, **kwargs): return self._root.get('cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def apps(self, **kwargs): return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def app(self, app_id): return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def kill(self, app_id): return self._root.put('cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE)
class ResourceManagerApi(object): def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False): self._url = posixpath.join(oozie_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled self._ssl_cert_ca_verify = ssl_cert_ca_verify if self._security_enabled: self._client.set_kerberos_auth() if ssl_cert_ca_verify: self._client.set_verify(True) def __str__(self): return "ResourceManagerApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def cluster(self, **kwargs): return self._root.get('cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def apps(self, **kwargs): return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def app(self, app_id): return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def kill(self, app_id): return self._root.put('cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE)
class OozieApi(object): def __init__(self, oozie_url, user, security_enabled=False, api_version=API_VERSION, ssl_cert_ca_verify=True): self._url = posixpath.join(oozie_url, api_version) self._client = HttpClient(self._url, logger=LOG) if security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) self._root = Resource(self._client) self._security_enabled = security_enabled # To store username info if hasattr(user, 'username'): self.user = user.username else: self.user = user self.api_version = api_version def __str__(self): return "OozieApi at %s" % (self._url, ) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def _get_params(self): if self.security_enabled: return {'doAs': self.user, 'timezone': TIME_ZONE.get()} return { 'user.name': DEFAULT_USER, 'doAs': self.user, 'timezone': TIME_ZONE.get() } def _get_oozie_properties(self, properties=None): defaults = { 'user.name': self.user, } if properties is not None: defaults.update(properties) return defaults VALID_JOB_FILTERS = ('name', 'user', 'group', 'status', 'startcreatedtime', 'text') VALID_LOG_FILTERS = set(('recent', 'limit', 'loglevel', 'text')) def get_jobs(self, jobtype, offset=None, cnt=None, filters=None): """ Get a list of Oozie jobs. Note that offset is 1-based. kwargs is used for filtering and may be one of VALID_FILTERS: name, user, group, status """ params = self._get_params() if offset is not None: params['offset'] = str(offset) if cnt is not None: params['len'] = str(cnt) if filters is None: filters = [] params['jobtype'] = jobtype filter_list = [] for key, val in filters: if key not in OozieApi.VALID_JOB_FILTERS: raise ValueError( '"%s" is not a valid filter for selecting jobs' % (key, )) filter_list.append('%s=%s' % (key, val)) params['filter'] = ';'.join(filter_list) # Send the request resp = self._root.get('jobs', params) if jobtype == 'wf': wf_list = WorkflowList(self, resp, filters=filters) elif jobtype == 'coord': wf_list = CoordinatorList(self, resp, filters=filters) else: wf_list = BundleList(self, resp, filters=filters) return wf_list def get_workflows(self, offset=None, cnt=None, filters=None): return self.get_jobs('wf', offset, cnt, filters) def get_coordinators(self, offset=None, cnt=None, filters=None): return self.get_jobs('coord', offset, cnt, filters) def get_bundles(self, offset=None, cnt=None, filters=None): return self.get_jobs('bundle', offset, cnt, filters) # TODO: make get_job accept any jobid def get_job(self, jobid): """ get_job(jobid) -> Workflow """ params = self._get_params() resp = self._root.get('job/%s' % (jobid, ), params) wf = Workflow(self, resp) return wf def get_coordinator(self, jobid, offset=None, cnt=None, filters=None): params = self._get_params() if offset is not None: params['offset'] = str(offset) if cnt is not None: params['len'] = str(cnt) if filters is None: filters = {} params.update({'order': 'desc'}) filter_list = [] for key, val in filters: if key not in OozieApi.VALID_JOB_FILTERS: raise ValueError( '"%s" is not a valid filter for selecting jobs' % (key, )) filter_list.append('%s=%s' % (key, val)) params['filter'] = ';'.join(filter_list) resp = self._root.get('job/%s' % (jobid, ), params) return Coordinator(self, resp) def get_bundle(self, jobid): params = self._get_params() resp = self._root.get('job/%s' % (jobid, ), params) return Bundle(self, resp) def get_job_definition(self, jobid): """ get_job_definition(jobid) -> Definition (xml string) """ params = self._get_params() params['show'] = 'definition' return self._root.get('job/%s' % (jobid, ), params) def get_job_log(self, jobid, logfilter=None): """ get_job_log(jobid) -> Log (xml string) """ params = self._get_params() params['show'] = 'log' filter_list = [] if logfilter is None: logfilter = [] for key, val in logfilter: if key not in OozieApi.VALID_LOG_FILTERS: raise ValueError('"%s" is not a valid filter for job logs' % (key, )) filter_list.append('%s=%s' % (key, val)) params['logfilter'] = ';'.join(filter_list) return self._root.get('job/%s' % (jobid, ), params) def get_job_status(self, jobid): params = self._get_params() params['show'] = 'status' xml = self._root.get('job/%s' % (jobid, ), params) return xml def get_action(self, action_id): if 'C@' in action_id: Klass = CoordinatorAction elif 'B@' in action_id: Klass = BundleAction else: Klass = WorkflowAction params = self._get_params() resp = self._root.get('job/%s' % (action_id, ), params) return Klass(resp) def job_control(self, jobid, action, properties=None, parameters=None): """ job_control(jobid, action) -> None Raise RestException on error. """ if action not in ('start', 'suspend', 'resume', 'kill', 'rerun', 'coord-rerun', 'bundle-rerun', 'change', 'ignore', 'update'): msg = 'Invalid oozie job action: %s' % (action, ) LOG.error(msg) raise ValueError(msg) properties = self._get_oozie_properties(properties) params = self._get_params() params['action'] = action if parameters is not None: params.update(parameters) return self._root.put('job/%s' % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) def submit_workflow(self, application_path, properties=None): """ submit_workflow(application_path, properties=None) -> jobid Raise RestException on error. """ defaults = { 'oozie.wf.application.path': application_path, 'user.name': self.user, } if properties is not None: defaults.update(properties) properties = defaults return self.submit_job(properties) # Is name actually submit_coord? def submit_job(self, properties=None): """ submit_job(properties=None, id=None) -> jobid Raise RestException on error. """ defaults = { 'user.name': self.user, } if properties is not None: defaults.update(properties) properties = defaults params = self._get_params() resp = self._root.post('jobs', params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) return resp['id'] def dryrun(self, properties=None): defaults = { 'user.name': self.user, } if properties is not None: defaults.update(properties) properties = defaults params = self._get_params() params['action'] = 'dryrun' return self._root.post('jobs', params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) def rerun(self, jobid, properties=None, params=None): properties = self._get_oozie_properties(properties) if params is None: params = self._get_params() else: self._get_params().update(params) params['action'] = 'rerun' return self._root.put('job/%s' % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) def get_build_version(self): """ get_build_version() -> Build version (dictionary) """ params = self._get_params() resp = self._root.get('admin/build-version', params) return resp def get_instrumentation(self): params = self._get_params() resp = self._root.get('admin/instrumentation', params) return resp def get_metrics(self): params = self._get_params() resp = self._root.get('admin/metrics', params) return resp def get_configuration(self): """ get_configuration() -> Oozie config (dictionary) """ params = self._get_params() resp = self._root.get('admin/configuration', params) return resp def get_oozie_status(self): """ get_oozie_status() -> Oozie status (dictionary) """ params = self._get_params() resp = self._root.get('admin/status', params) return resp def get_oozie_slas(self, **kwargs): """ filter= app_name=my-sla-app id=0000002-131206135002457-oozie-oozi-W nominal_start=2013-06-18T00:01Z nominal_end=2013-06-23T00:01Z """ params = self._get_params() params['filter'] = ';'.join( ['%s=%s' % (key, val) for key, val in kwargs.iteritems()]) resp = self._root.get('sla', params) return resp['slaSummaryList']
class OozieApi(object): def __init__(self, oozie_url, security_enabled=False): self._url = posixpath.join(oozie_url, API_VERSION) self._client = HttpClient(self._url, logger=LOG) if security_enabled: self._client.set_kerberos_auth() self._root = Resource(self._client) self._security_enabled = security_enabled # To store user info self._thread_local = threading.local() def __str__(self): return "OozieApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def user(self): try: return self._thread_local.user except AttributeError: return DEFAULT_USER def setuser(self, user): """Return the previous user""" prev = self.user self._thread_local.user = user return prev def _get_params(self): if self.security_enabled: return { 'doAs': self.user, 'timezone': TIME_ZONE.get() } return { 'user.name': DEFAULT_USER, 'doAs': self.user, 'timezone': TIME_ZONE.get() } def _get_oozie_properties(self, properties=None): defaults = { 'user.name': self.user, } if properties is not None: defaults.update(properties) return defaults VALID_JOB_FILTERS = ('name', 'user', 'group', 'status') def get_jobs(self, jobtype, offset=None, cnt=None, **kwargs): """ Get a list of Oozie jobs. jobtype is 'wf', 'coord' Note that offset is 1-based. kwargs is used for filtering and may be one of VALID_FILTERS: name, user, group, status """ params = self._get_params() if offset is not None: params['offset'] = str(offset) if cnt is not None: params['len'] = str(cnt) params['jobtype'] = jobtype filter_list = [ ] for key, val in kwargs.iteritems(): if key not in OozieApi.VALID_JOB_FILTERS: raise ValueError('"%s" is not a valid filter for selecting jobs' % (key,)) filter_list.append('%s=%s' % (key, val)) params['filter'] = ';'.join(filter_list) # Send the request resp = self._root.get('jobs', params) if jobtype == 'wf': wf_list = WorkflowList(self, resp, filters=kwargs) elif jobtype == 'coord': wf_list = CoordinatorList(self, resp, filters=kwargs) else: wf_list = BundleList(self, resp, filters=kwargs) return wf_list def get_workflows(self, offset=None, cnt=None, **kwargs): return self.get_jobs('wf', offset, cnt, **kwargs) def get_coordinators(self, offset=None, cnt=None, **kwargs): return self.get_jobs('coord', offset, cnt, **kwargs) def get_bundles(self, offset=None, cnt=None, **kwargs): return self.get_jobs('bundle', offset, cnt, **kwargs) # TODO: make get_job accept any jobid def get_job(self, jobid): """ get_job(jobid) -> Workflow """ params = self._get_params() resp = self._root.get('job/%s' % (jobid,), params) wf = Workflow(self, resp) return wf def get_coordinator(self, jobid): params = self._get_params() params.update({'len': -1}) resp = self._root.get('job/%s' % (jobid,), params) return Coordinator(self, resp) def get_bundle(self, jobid): params = self._get_params() resp = self._root.get('job/%s' % (jobid,), params) return Bundle(self, resp) def get_job_definition(self, jobid): """ get_job_definition(jobid) -> Definition (xml string) """ params = self._get_params() params['show'] = 'definition' xml = self._root.get('job/%s' % (jobid,), params) return xml def get_job_log(self, jobid): """ get_job_log(jobid) -> Log (xml string) """ params = self._get_params() params['show'] = 'log' xml = self._root.get('job/%s' % (jobid,), params) return xml def get_action(self, action_id): if 'C@' in action_id: Klass = CoordinatorAction elif 'B@' in action_id: Klass = BundleAction else: Klass = WorkflowAction params = self._get_params() resp = self._root.get('job/%s' % (action_id,), params) return Klass(resp) def job_control(self, jobid, action, properties=None, parameters=None): """ job_control(jobid, action) -> None Raise RestException on error. """ if action not in ('start', 'suspend', 'resume', 'kill', 'rerun', 'coord-rerun', 'bundle-rerun'): msg = 'Invalid oozie job action: %s' % (action,) LOG.error(msg) raise ValueError(msg) properties = self._get_oozie_properties(properties) params = self._get_params() params['action'] = action if parameters is not None: params.update(parameters) return self._root.put('job/%s' % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) def submit_workflow(self, application_path, properties=None): """ submit_workflow(application_path, properties=None) -> jobid Raise RestException on error. """ defaults = { 'oozie.wf.application.path': application_path, 'user.name': self.user, } if properties is not None: defaults.update(properties) properties = defaults return self.submit_job(properties) # Is name actually submit_coord? def submit_job(self, properties=None): """ submit_job(properties=None, id=None) -> jobid Raise RestException on error. """ defaults = { 'user.name': self.user, } if properties is not None: defaults.update(properties) properties = defaults params = self._get_params() resp = self._root.post('jobs', params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) return resp['id'] def rerun(self, jobid, properties=None, params=None): properties = self._get_oozie_properties(properties) if params is None: params = self._get_params() else: self._get_params().update(params) params['action'] = 'rerun' return self._root.put('job/%s' % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) def get_build_version(self): """ get_build_version() -> Build version (dictionary) """ params = self._get_params() resp = self._root.get('admin/build-version', params) return resp def get_instrumentation(self): params = self._get_params() resp = self._root.get('admin/instrumentation', params) return resp def get_configuration(self): """ get_configuration() -> Oozie config (dictionary) """ params = self._get_params() resp = self._root.get('admin/configuration', params) return resp def get_oozie_status(self): """ get_oozie_status() -> Oozie status (dictionary) """ params = self._get_params() resp = self._root.get('admin/status', params) return resp
class OozieApi(object): def __init__(self, oozie_url, security_enabled=False, api_version=API_VERSION): self._url = posixpath.join(oozie_url, api_version) self._client = HttpClient(self._url, logger=LOG) if security_enabled: self._client.set_kerberos_auth() self._root = Resource(self._client) self._security_enabled = security_enabled # To store username info self._thread_local = threading.local() self.api_version = api_version def __str__(self): return "OozieApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def user(self): return self._thread_local.user def setuser(self, user): if hasattr(user, "username"): self._thread_local.user = user.username else: self._thread_local.user = user def _get_params(self): if self.security_enabled: return {"doAs": self.user, "timezone": TIME_ZONE.get()} return {"user.name": DEFAULT_USER, "doAs": self.user, "timezone": TIME_ZONE.get()} def _get_oozie_properties(self, properties=None): defaults = {"user.name": self.user} if properties is not None: defaults.update(properties) return defaults VALID_JOB_FILTERS = ("name", "user", "group", "status") def get_jobs(self, jobtype, offset=None, cnt=None, **kwargs): """ Get a list of Oozie jobs. jobtype is 'wf', 'coord' Note that offset is 1-based. kwargs is used for filtering and may be one of VALID_FILTERS: name, user, group, status """ params = self._get_params() if offset is not None: params["offset"] = str(offset) if cnt is not None: params["len"] = str(cnt) params["jobtype"] = jobtype filter_list = [] for key, val in kwargs.iteritems(): if key not in OozieApi.VALID_JOB_FILTERS: raise ValueError('"%s" is not a valid filter for selecting jobs' % (key,)) filter_list.append("%s=%s" % (key, val)) params["filter"] = ";".join(filter_list) # Send the request resp = self._root.get("jobs", params) if jobtype == "wf": wf_list = WorkflowList(self, resp, filters=kwargs) elif jobtype == "coord": wf_list = CoordinatorList(self, resp, filters=kwargs) else: wf_list = BundleList(self, resp, filters=kwargs) return wf_list def get_workflows(self, offset=None, cnt=None, **kwargs): return self.get_jobs("wf", offset, cnt, **kwargs) def get_coordinators(self, offset=None, cnt=None, **kwargs): return self.get_jobs("coord", offset, cnt, **kwargs) def get_bundles(self, offset=None, cnt=None, **kwargs): return self.get_jobs("bundle", offset, cnt, **kwargs) # TODO: make get_job accept any jobid def get_job(self, jobid): """ get_job(jobid) -> Workflow """ params = self._get_params() resp = self._root.get("job/%s" % (jobid,), params) wf = Workflow(self, resp) return wf def get_coordinator(self, jobid): params = self._get_params() params.update({"len": -1}) resp = self._root.get("job/%s" % (jobid,), params) return Coordinator(self, resp) def get_bundle(self, jobid): params = self._get_params() resp = self._root.get("job/%s" % (jobid,), params) return Bundle(self, resp) def get_job_definition(self, jobid): """ get_job_definition(jobid) -> Definition (xml string) """ params = self._get_params() params["show"] = "definition" xml = self._root.get("job/%s" % (jobid,), params) return xml def get_job_log(self, jobid): """ get_job_log(jobid) -> Log (xml string) """ params = self._get_params() params["show"] = "log" xml = self._root.get("job/%s" % (jobid,), params) return xml def get_action(self, action_id): if "C@" in action_id: Klass = CoordinatorAction elif "B@" in action_id: Klass = BundleAction else: Klass = WorkflowAction params = self._get_params() resp = self._root.get("job/%s" % (action_id,), params) return Klass(resp) def job_control(self, jobid, action, properties=None, parameters=None): """ job_control(jobid, action) -> None Raise RestException on error. """ if action not in ("start", "suspend", "resume", "kill", "rerun", "coord-rerun", "bundle-rerun"): msg = "Invalid oozie job action: %s" % (action,) LOG.error(msg) raise ValueError(msg) properties = self._get_oozie_properties(properties) params = self._get_params() params["action"] = action if parameters is not None: params.update(parameters) return self._root.put("job/%s" % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) def submit_workflow(self, application_path, properties=None): """ submit_workflow(application_path, properties=None) -> jobid Raise RestException on error. """ defaults = {"oozie.wf.application.path": application_path, "user.name": self.user} if properties is not None: defaults.update(properties) properties = defaults return self.submit_job(properties) # Is name actually submit_coord? def submit_job(self, properties=None): """ submit_job(properties=None, id=None) -> jobid Raise RestException on error. """ defaults = {"user.name": self.user} if properties is not None: defaults.update(properties) properties = defaults params = self._get_params() resp = self._root.post("jobs", params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) return resp["id"] def rerun(self, jobid, properties=None, params=None): properties = self._get_oozie_properties(properties) if params is None: params = self._get_params() else: self._get_params().update(params) params["action"] = "rerun" return self._root.put("job/%s" % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) def get_build_version(self): """ get_build_version() -> Build version (dictionary) """ params = self._get_params() resp = self._root.get("admin/build-version", params) return resp def get_instrumentation(self): params = self._get_params() resp = self._root.get("admin/instrumentation", params) return resp def get_configuration(self): """ get_configuration() -> Oozie config (dictionary) """ params = self._get_params() resp = self._root.get("admin/configuration", params) return resp def get_oozie_status(self): """ get_oozie_status() -> Oozie status (dictionary) """ params = self._get_params() resp = self._root.get("admin/status", params) return resp def get_oozie_slas(self, **kwargs): """ filter= app_name=my-sla-app id=0000002-131206135002457-oozie-oozi-W nominal_start=2013-06-18T00:01Z nominal_end=2013-06-23T00:01Z """ params = self._get_params() params["filter"] = ";".join(["%s=%s" % (key, val) for key, val in kwargs.iteritems()]) resp = self._root.get("sla", params) return resp["slaSummaryList"]
class OozieApi(object): def __init__(self, oozie_url, user, security_enabled=False, api_version=API_VERSION, ssl_cert_ca_verify=True): self._url = posixpath.join(oozie_url, api_version) self._client = HttpClient(self._url, logger=LOG) if security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) self._root = Resource(self._client) self._security_enabled = security_enabled # To store username info if hasattr(user, 'username'): self.user = user.username else: self.user = user self.api_version = api_version def __str__(self): return "OozieApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def _get_params(self): if self.security_enabled: return { 'doAs': self.user, 'timezone': TIME_ZONE.get() } return { 'user.name': DEFAULT_USER, 'doAs': self.user, 'timezone': TIME_ZONE.get() } def _get_oozie_properties(self, properties=None): defaults = { 'user.name': self.user, } if properties is not None: defaults.update(properties) return defaults VALID_JOB_FILTERS = ('name', 'user', 'group', 'status', 'startcreatedtime') VALID_LOG_FILTERS = {'recent', 'limit', 'loglevel', 'text'} def get_jobs(self, jobtype, offset=None, cnt=None, filters=None): """ Get a list of Oozie jobs. Note that offset is 1-based. kwargs is used for filtering and may be one of VALID_FILTERS: name, user, group, status """ params = self._get_params() if offset is not None: params['offset'] = str(offset) if cnt is not None: params['len'] = str(cnt) if filters is None: filters = [] params['jobtype'] = jobtype filter_list = [] for key, val in filters: if key not in OozieApi.VALID_JOB_FILTERS: raise ValueError('"%s" is not a valid filter for selecting jobs' % (key,)) filter_list.append('%s=%s' % (key, val)) params['filter'] = ';'.join(filter_list) # Send the request resp = self._root.get('jobs', params) if jobtype == 'wf': wf_list = WorkflowList(self, resp, filters=filters) elif jobtype == 'coord': wf_list = CoordinatorList(self, resp, filters=filters) else: wf_list = BundleList(self, resp, filters=filters) return wf_list def get_workflows(self, offset=None, cnt=None, filters=None): return self.get_jobs('wf', offset, cnt, filters) def get_coordinators(self, offset=None, cnt=None, filters=None): return self.get_jobs('coord', offset, cnt, filters) def get_bundles(self, offset=None, cnt=None, filters=None): return self.get_jobs('bundle', offset, cnt, filters) # TODO: make get_job accept any jobid def get_job(self, jobid): """ get_job(jobid) -> Workflow """ params = self._get_params() resp = self._root.get('job/%s' % (jobid,), params) wf = Workflow(self, resp) return wf def get_coordinator(self, jobid, offset=None, cnt=None, filters=None): params = self._get_params() if offset is not None: params['offset'] = str(offset) if cnt is not None: params['len'] = str(cnt) if filters is None: filters = {} params.update({'order': 'desc'}) filter_list = [] for key, val in filters: if key not in OozieApi.VALID_JOB_FILTERS: raise ValueError('"%s" is not a valid filter for selecting jobs' % (key,)) filter_list.append('%s=%s' % (key, val)) params['filter'] = ';'.join(filter_list) resp = self._root.get('job/%s' % (jobid,), params) return Coordinator(self, resp) def get_bundle(self, jobid): params = self._get_params() resp = self._root.get('job/%s' % (jobid,), params) return Bundle(self, resp) def get_job_definition(self, jobid): """ get_job_definition(jobid) -> Definition (xml string) """ params = self._get_params() params['show'] = 'definition' return self._root.get('job/%s' % (jobid,), params) def get_job_log(self, jobid, logfilter=None): """ get_job_log(jobid) -> Log (xml string) """ params = self._get_params() params['show'] = 'log' filter_list = [] if logfilter is None: logfilter = [] for key, val in logfilter: if key not in OozieApi.VALID_LOG_FILTERS: raise ValueError('"%s" is not a valid filter for job logs' % (key,)) filter_list.append('%s=%s' % (key, val)) params['logfilter'] = ';'.join(filter_list) return self._root.get('job/%s' % (jobid,), params) def get_job_status(self, jobid): params = self._get_params() params['show'] = 'status' xml = self._root.get('job/%s' % (jobid,), params) return xml def get_action(self, action_id): if 'C@' in action_id: Klass = CoordinatorAction elif 'B@' in action_id: Klass = BundleAction else: Klass = WorkflowAction params = self._get_params() resp = self._root.get('job/%s' % (action_id,), params) return Klass(resp) def job_control(self, jobid, action, properties=None, parameters=None): """ job_control(jobid, action) -> None Raise RestException on error. """ if action not in ('start', 'suspend', 'resume', 'kill', 'rerun', 'coord-rerun', 'bundle-rerun', 'change', 'ignore'): msg = 'Invalid oozie job action: %s' % (action,) LOG.error(msg) raise ValueError(msg) properties = self._get_oozie_properties(properties) params = self._get_params() params['action'] = action if parameters is not None: params.update(parameters) return self._root.put('job/%s' % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) def submit_workflow(self, application_path, properties=None): """ submit_workflow(application_path, properties=None) -> jobid Raise RestException on error. """ defaults = { 'oozie.wf.application.path': application_path, 'user.name': self.user, } if properties is not None: defaults.update(properties) properties = defaults return self.submit_job(properties) # Is name actually submit_coord? def submit_job(self, properties=None): """ submit_job(properties=None, id=None) -> jobid Raise RestException on error. """ defaults = { 'user.name': self.user, } if properties is not None: defaults.update(properties) properties = defaults params = self._get_params() resp = self._root.post('jobs', params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) return resp['id'] def dryrun(self, properties=None): defaults = { 'user.name': self.user, } if properties is not None: defaults.update(properties) properties = defaults params = self._get_params() params['action'] = 'dryrun' return self._root.post('jobs', params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) def rerun(self, jobid, properties=None, params=None): properties = self._get_oozie_properties(properties) if params is None: params = self._get_params() else: self._get_params().update(params) params['action'] = 'rerun' return self._root.put('job/%s' % jobid, params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) def get_build_version(self): """ get_build_version() -> Build version (dictionary) """ params = self._get_params() resp = self._root.get('admin/build-version', params) return resp def get_instrumentation(self): params = self._get_params() resp = self._root.get('admin/instrumentation', params) return resp def get_metrics(self): params = self._get_params() resp = self._root.get('admin/metrics', params) return resp def get_configuration(self): """ get_configuration() -> Oozie config (dictionary) """ params = self._get_params() resp = self._root.get('admin/configuration', params) return resp def get_oozie_status(self): """ get_oozie_status() -> Oozie status (dictionary) """ params = self._get_params() resp = self._root.get('admin/status', params) return resp def get_oozie_slas(self, **kwargs): """ filter= app_name=my-sla-app id=0000002-131206135002457-oozie-oozi-W nominal_start=2013-06-18T00:01Z nominal_end=2013-06-23T00:01Z """ params = self._get_params() params['filter'] = ';'.join(['%s=%s' % (key, val) for key, val in kwargs.iteritems()]) resp = self._root.get('sla', params) return resp['slaSummaryList']
class OozieApi(object): def __init__(self, oozie_url, security_enabled=False): self._url = posixpath.join(oozie_url, API_VERSION) self._client = HttpClient(self._url, logger=LOG) if security_enabled: self._client.set_kerberos_auth() self._root = Resource(self._client) self._security_enabled = security_enabled # To store user info self._thread_local = threading.local() def __str__(self): return "OozieApi at %s" % (self._url,) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled @property def user(self): try: return self._thread_local.user except AttributeError: return DEFAULT_USER def setuser(self, user): """Return the previous user""" prev = self.user self._thread_local.user = user return prev def _get_params(self): if self.security_enabled: return {"doAs": self.user} return {"user.name": DEFAULT_USER, "doAs": self.user} VALID_JOB_FILTERS = ("name", "user", "group", "status") def get_jobs(self, offset=None, cnt=None, **kwargs): """ get_jobs(offset=None, cnt=None, **kwargs) -> WorkflowList Note that offset is 1-based. kwargs is used for filtering and may be one of VALID_FILTERS: name, user, group, status """ params = self._get_params() if offset is not None: params["offset"] = str(offset) if cnt is not None: params["len"] = str(cnt) filter_list = [] for key, val in kwargs: if key not in OozieApi.VALID_JOB_FILTERS: raise ValueError('"%s" is not a valid filter for selecting jobs' % (key,)) filter_list.append("%s=%s" % (key, val)) params["filter"] = ";".join(filter_list) # Send the request resp = self._root.get("jobs", params) wf_list = WorkflowList(self, resp, filters=kwargs) return wf_list def get_job(self, jobid): """ get_job(jobid) -> Workflow """ params = self._get_params() resp = self._root.get("job/%s" % (jobid,), params) wf = Workflow(self, resp) return wf def get_job_definition(self, jobid): """ get_job_definition(jobid) -> Definition (xml string) """ params = self._get_params() params["show"] = "definition" xml = self._root.get("job/%s" % (jobid,), params) return xml def get_job_log(self, jobid): """ get_job_log(jobid) -> Log (xml string) """ params = self._get_params() params["show"] = "log" xml = self._root.get("job/%s" % (jobid,), params) return xml def job_control(self, jobid, action): """ job_control(jobid, action) -> None Raise RestException on error. """ if action not in ("start", "suspend", "resume", "kill"): msg = "Invalid oozie job action: %s" % (action,) LOG.error(msg) raise ValueError(msg) params = self._get_params() params["action"] = action self._root.put("job/%s" % (jobid,), params) def submit_workflow(self, application_path, properties=None): """ submit_workflow(application_path, username, properties=None) -> jobid Submit a job to Oozie. May raise PopupException. """ defaults = {"oozie.wf.application.path": application_path, "user.name": self.user} if properties is not None: defaults.update(properties) properties = defaults else: properties = defaults params = self._get_params() resp = self._root.post("jobs", params, data=config_gen(properties), contenttype=_XML_CONTENT_TYPE) return resp["id"] def get_build_version(self): """ get_build_version() -> Build version (dictionary) """ params = self._get_params() resp = self._root.get("admin/build-version", params) return resp def get_instrumentation(self): """ get_instrumentation() -> Oozie instrumentation (dictionary) """ params = self._get_params() resp = self._root.get("admin/instrumentation", params) return resp def get_configuration(self): """ get_configuration() -> Oozie config (dictionary) """ params = self._get_params() resp = self._root.get("admin/configuration", params) return resp def get_oozie_status(self): """ get_oozie_status() -> Oozie status (dictionary) """ params = self._get_params() resp = self._root.get("admin/status", params) return resp
class ResourceManagerApi(object): def __init__(self, oozie_url, security_enabled=False, ssl_cert_ca_verify=False): self._url = posixpath.join(oozie_url, 'ws', _API_VERSION) self._client = HttpClient(self._url, logger=LOG) self._root = Resource(self._client) self._security_enabled = security_enabled if self._security_enabled: self._client.set_kerberos_auth() self._client.set_verify(ssl_cert_ca_verify) def __str__(self): return "ResourceManagerApi at %s" % (self._url, ) @property def url(self): return self._url @property def security_enabled(self): return self._security_enabled def cluster(self, **kwargs): return self._root.get('cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) return self._execute(self._root.get, 'cluster', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def apps(self, **kwargs): return self._root.get('cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) return self._execute(self._root.get, 'cluster/apps', params=kwargs, headers={'Accept': _JSON_CONTENT_TYPE}) def app(self, app_id): return self._root.get('cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE}) return self._execute(self._root.get, 'cluster/apps/%(app_id)s' % {'app_id': app_id}, headers={'Accept': _JSON_CONTENT_TYPE}) def kill(self, app_id): return self._root.put('cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE) return self._execute(self._root.put, 'cluster/apps/%(app_id)s/state' % {'app_id': app_id}, data=json.dumps({'state': 'KILLED'}), contenttype=_JSON_CONTENT_TYPE) def _execute(self, function, *args, **kwargs): response = function(*args, **kwargs) # YARN-2605: Yarn does not use proper HTTP redirects when the standby RM has # failed back to the master RM. if isinstance(response, str) and response.startswith( 'This is standby RM. Redirecting to the current active RM'): raise YarnFailoverOccurred(response) return response