def __init__(self, user): self.user = user self.resource_manager_api = resource_manager_api.get_resource_manager( user.username) self.mapreduce_api = mapreduce_api.get_mapreduce_api(user.username) self.history_server_api = history_server_api.get_history_server_api() self.spark_history_server_api = spark_history_server_api.get_history_server_api( )
def app(self, appid): try: job = NativeYarnApi(self.user).get_job(jobid=appid) except ApplicationNotRunning, e: if e.job.get('state', '').lower() == 'accepted': rm_api = resource_manager_api.get_resource_manager(self.user) job = Application(e.job, rm_api) else: raise e # Job has not yet been accepted by RM
def get_job(request, job_id): try: job = get_api(request.user, request.jt).get_job(jobid=job_id) except ApplicationNotRunning, e: if e.job.get('state', '').lower() == 'accepted': rm_api = resource_manager_api.get_resource_manager(request.user) job = Application(e.job, rm_api) else: raise e # Job has not yet been accepted by RM
def __init__(self, user): self.user = user self.resource_manager_api = None self.mapreduce_api = None self.history_server_api = None self.spark_history_server_api = None if list(YARN_CLUSTERS.keys()): self.resource_manager_api = resource_manager_api.get_resource_manager(user.username) self.mapreduce_api = mapreduce_api.get_mapreduce_api(user.username) self.history_server_api = history_server_api.get_history_server_api(user.username) self.spark_history_server_api = spark_history_server_api.get_history_server_api() # Spark HS does not support setuser
def decorate(request, *args, **kwargs): jobid = kwargs['job'] try: job = get_api(request.user, request.jt).get_job(jobid=jobid) except ApplicationNotRunning, e: if e.job.get('state', '').lower() == 'accepted' and 'kill' in request.path: rm_api = resource_manager_api.get_resource_manager(request.user) job = Application(e.job, rm_api) else: # reverse() seems broken, using request.path but beware, it discards GET and POST info return job_not_assigned(request, jobid, request.path)
def test_yarn_configurations(): # Single cluster for now result = [] try: url = '' api = get_resource_manager() url = api._url api.apps() except Exception, e: msg = 'Failed to contact Resource Manager at %s: %s' % (url, e) result.append(('Resource Manager', msg))
def test_yarn_configurations(user): # Single cluster for now from hadoop.yarn.resource_manager_api import get_resource_manager result = [] try: url = '' api = get_resource_manager(user.username) url = api._url api.apps() except Exception, e: msg = 'Failed to contact Resource Manager at %s: %s' % (url, e) LOG.exception(msg) result.append(('Resource Manager', msg))
def get_job(request, job_id): try: job = get_api(request.user, request.jt).get_job(jobid=job_id) except ApplicationNotRunning as e: if e.job.get('state', '').lower() == 'accepted': rm_api = resource_manager_api.get_resource_manager(request.user) job = Application(e.job, rm_api) else: raise e # Job has not yet been accepted by RM except JobExpired as e: raise PopupException( _('Job %s has expired.') % job_id, detail=_('Cannot be found on the History Server.')) except Exception as e: msg = 'Could not find job %s.' LOG.exception(msg % job_id) raise PopupException(_(msg) % job_id, detail=e) return job
def __init__(self, user): self.user = user self.resource_manager_api = resource_manager_api.get_resource_manager(user.username) self.mapreduce_api = mapreduce_api.get_mapreduce_api(user.username) self.history_server_api = history_server_api.get_history_server_api(user.username) self.spark_history_server_api = spark_history_server_api.get_history_server_api()
def app(self, appid): try: job = NativeYarnApi(self.user).get_job(jobid=appid) except ApplicationNotRunning as e: if e.job.get('state', '').lower() == 'accepted': rm_api = resource_manager_api.get_resource_manager(self.user) job = Application(e.job, rm_api) else: raise e # Job has not yet been accepted by RM except JobExpired as e: raise PopupException(_('Job %s has expired.') % appid, detail=_('Cannot be found on the History Server.')) except Exception as e: msg = 'Could not find job %s.' LOG.exception(msg % appid) raise PopupException(_(msg) % appid, detail=e) app = massage_job_for_json(job, user=self.user) common = { 'id': app['id'], 'name': app['name'], 'type': app['applicationType'], 'status': app['status'], 'apiStatus': self._api_status(app['status']), 'user': app['user'], 'progress': app['progress'], 'duration': app['durationMs'], 'submitted': app['startTimeMs'], 'canWrite': app['canKill'], } if app['applicationType'] == 'MR2' or app['applicationType'] == 'MAPREDUCE': common['type'] = 'MAPREDUCE' if app['desiredMaps'] is None or app['finishedMaps'] is None: app['mapsPercentComplete'] = 100 if app['desiredReduces'] is None or app['finishedReduces'] is None: app['reducesPercentComplete'] = 100 common['properties'] = { 'maps_percent_complete': app['mapsPercentComplete'] or 0, 'reduces_percent_complete': app['reducesPercentComplete'] or 0, 'finishedMaps': app['finishedMaps'] or 0, 'finishedReduces': app['finishedReduces'] or 0, 'desiredMaps': app['desiredMaps'] or 0, 'desiredReduces': app['desiredReduces'] or 0, 'durationFormatted': app['durationFormatted'], 'startTimeFormatted': app['startTimeFormatted'], 'diagnostics': app['diagnostics'] if app['diagnostics'] else '', 'tasks': [], 'metadata': [], 'counters': [] } elif app['applicationType'] == 'SPARK': app['logs'] = job.logs_url if hasattr(job, 'logs_url') else '' app['trackingUrl'] = job.trackingUrl if hasattr(job, 'trackingUrl') else '' common['type'] = 'SPARK' common['properties'] = { 'metadata': [{'name': name, 'value': value} for name, value in app.items() if name != "url" and name != "killUrl"], 'executors': [] } if hasattr(job, 'metrics'): common['metrics'] = job.metrics elif app['applicationType'] == 'YarnV2': common['applicationType'] = app.get('type') common['properties'] = { 'startTime': job.startTime, 'finishTime': job.finishTime, 'elapsedTime': job.duration, 'attempts': [], 'diagnostics': job.diagnostics } return common
def kill(self, job_id): app_id = job_id.replace('job', 'application') get_resource_manager().kill(app_id) # We need to call the RM
def __init__(self, user): self.user = user self.resource_manager_api = resource_manager_api.get_resource_manager() self.mapreduce_api = mapreduce_api.get_mapreduce_api() self.node_manager_api = node_manager_api.get_resource_manager_api() self.history_server_api = history_server_api.get_history_server_api()