Example #1
0
File: api.py Project: eahagopi/hue
 def __init__(self, user):
     self.user = user
     self.resource_manager_api = resource_manager_api.get_resource_manager(
         user.username)
     self.mapreduce_api = mapreduce_api.get_mapreduce_api(user.username)
     self.history_server_api = history_server_api.get_history_server_api()
     self.spark_history_server_api = spark_history_server_api.get_history_server_api(
     )
Example #2
0
 def app(self, appid):
     try:
         job = NativeYarnApi(self.user).get_job(jobid=appid)
     except ApplicationNotRunning, e:
         if e.job.get('state', '').lower() == 'accepted':
             rm_api = resource_manager_api.get_resource_manager(self.user)
             job = Application(e.job, rm_api)
         else:
             raise e  # Job has not yet been accepted by RM
Example #3
0
def get_job(request, job_id):
  try:
    job = get_api(request.user, request.jt).get_job(jobid=job_id)
  except ApplicationNotRunning, e:
    if e.job.get('state', '').lower() == 'accepted':
      rm_api = resource_manager_api.get_resource_manager(request.user)
      job = Application(e.job, rm_api)
    else:
      raise e  # Job has not yet been accepted by RM
Example #4
0
def get_job(request, job_id):
    try:
        job = get_api(request.user, request.jt).get_job(jobid=job_id)
    except ApplicationNotRunning, e:
        if e.job.get('state', '').lower() == 'accepted':
            rm_api = resource_manager_api.get_resource_manager(request.user)
            job = Application(e.job, rm_api)
        else:
            raise e  # Job has not yet been accepted by RM
Example #5
0
 def app(self, appid):
   try:
     job = NativeYarnApi(self.user).get_job(jobid=appid)
   except ApplicationNotRunning, e:
     if e.job.get('state', '').lower() == 'accepted':
       rm_api = resource_manager_api.get_resource_manager(self.user)
       job = Application(e.job, rm_api)
     else:
       raise e  # Job has not yet been accepted by RM
Example #6
0
File: api.py Project: ranade1/hue-3
 def __init__(self, user):
   self.user = user
   self.resource_manager_api = None
   self.mapreduce_api = None
   self.history_server_api = None
   self.spark_history_server_api = None
   if list(YARN_CLUSTERS.keys()):
     self.resource_manager_api = resource_manager_api.get_resource_manager(user.username)
     self.mapreduce_api = mapreduce_api.get_mapreduce_api(user.username)
     self.history_server_api = history_server_api.get_history_server_api(user.username)
     self.spark_history_server_api = spark_history_server_api.get_history_server_api()  # Spark HS does not support setuser
Example #7
0
 def decorate(request, *args, **kwargs):
   jobid = kwargs['job']
   try:
     job = get_api(request.user, request.jt).get_job(jobid=jobid)
   except ApplicationNotRunning, e:
     if e.job.get('state', '').lower() == 'accepted' and 'kill' in request.path:
       rm_api = resource_manager_api.get_resource_manager(request.user)
       job = Application(e.job, rm_api)
     else:
       # reverse() seems broken, using request.path but beware, it discards GET and POST info
       return job_not_assigned(request, jobid, request.path)
Example #8
0
File: views.py Project: ronwxy/hue
 def decorate(request, *args, **kwargs):
   jobid = kwargs['job']
   try:
     job = get_api(request.user, request.jt).get_job(jobid=jobid)
   except ApplicationNotRunning, e:
     if e.job.get('state', '').lower() == 'accepted' and 'kill' in request.path:
       rm_api = resource_manager_api.get_resource_manager(request.user)
       job = Application(e.job, rm_api)
     else:
       # reverse() seems broken, using request.path but beware, it discards GET and POST info
       return job_not_assigned(request, jobid, request.path)
Example #9
0
def test_yarn_configurations():
  # Single cluster for now

  result = []

  try:
    url = ''
    api = get_resource_manager()
    url = api._url
    api.apps()
  except Exception, e:
    msg = 'Failed to contact Resource Manager at %s: %s' % (url, e)
    result.append(('Resource Manager', msg))
Example #10
0
def test_yarn_configurations():
    # Single cluster for now

    result = []

    try:
        url = ''
        api = get_resource_manager()
        url = api._url
        api.apps()
    except Exception, e:
        msg = 'Failed to contact Resource Manager at %s: %s' % (url, e)
        result.append(('Resource Manager', msg))
Example #11
0
File: conf.py Project: rosomaxa/hue
def test_yarn_configurations(user):
  # Single cluster for now
  from hadoop.yarn.resource_manager_api import get_resource_manager

  result = []

  try:
    url = ''
    api = get_resource_manager(user.username)
    url = api._url
    api.apps()
  except Exception, e:
    msg = 'Failed to contact Resource Manager at %s: %s' % (url, e)
    LOG.exception(msg)
    result.append(('Resource Manager', msg))
Example #12
0
def test_yarn_configurations(user):
    # Single cluster for now
    from hadoop.yarn.resource_manager_api import get_resource_manager

    result = []

    try:
        url = ''
        api = get_resource_manager(user.username)
        url = api._url
        api.apps()
    except Exception, e:
        msg = 'Failed to contact Resource Manager at %s: %s' % (url, e)
        LOG.exception(msg)
        result.append(('Resource Manager', msg))
Example #13
0
def get_job(request, job_id):
    try:
        job = get_api(request.user, request.jt).get_job(jobid=job_id)
    except ApplicationNotRunning as e:
        if e.job.get('state', '').lower() == 'accepted':
            rm_api = resource_manager_api.get_resource_manager(request.user)
            job = Application(e.job, rm_api)
        else:
            raise e  # Job has not yet been accepted by RM
    except JobExpired as e:
        raise PopupException(
            _('Job %s has expired.') % job_id,
            detail=_('Cannot be found on the History Server.'))
    except Exception as e:
        msg = 'Could not find job %s.'
        LOG.exception(msg % job_id)
        raise PopupException(_(msg) % job_id, detail=e)
    return job
Example #14
0
 def __init__(self, user):
   self.user = user
   self.resource_manager_api = resource_manager_api.get_resource_manager(user.username)
   self.mapreduce_api = mapreduce_api.get_mapreduce_api(user.username)
   self.history_server_api = history_server_api.get_history_server_api(user.username)
   self.spark_history_server_api = spark_history_server_api.get_history_server_api()
Example #15
0
  def app(self, appid):
    try:
      job = NativeYarnApi(self.user).get_job(jobid=appid)
    except ApplicationNotRunning as e:
      if e.job.get('state', '').lower() == 'accepted':
        rm_api = resource_manager_api.get_resource_manager(self.user)
        job = Application(e.job, rm_api)
      else:
        raise e  # Job has not yet been accepted by RM
    except JobExpired as e:
      raise PopupException(_('Job %s has expired.') % appid, detail=_('Cannot be found on the History Server.'))
    except Exception as e:
      msg = 'Could not find job %s.'
      LOG.exception(msg % appid)
      raise PopupException(_(msg) % appid, detail=e)


    app = massage_job_for_json(job, user=self.user)

    common = {
        'id': app['id'],
        'name': app['name'],
        'type': app['applicationType'],
        'status': app['status'],
        'apiStatus': self._api_status(app['status']),
        'user': app['user'],
        'progress': app['progress'],
        'duration': app['durationMs'],
        'submitted': app['startTimeMs'],
        'canWrite': app['canKill'],
    }

    if app['applicationType'] == 'MR2' or app['applicationType'] == 'MAPREDUCE':
      common['type'] = 'MAPREDUCE'

      if app['desiredMaps'] is None or app['finishedMaps'] is None:
        app['mapsPercentComplete'] = 100
      if app['desiredReduces'] is None or app['finishedReduces'] is None:
        app['reducesPercentComplete'] = 100

      common['properties'] = {
          'maps_percent_complete': app['mapsPercentComplete'] or 0,
          'reduces_percent_complete': app['reducesPercentComplete'] or 0,
          'finishedMaps': app['finishedMaps'] or 0,
          'finishedReduces': app['finishedReduces'] or 0,
          'desiredMaps': app['desiredMaps'] or 0,
          'desiredReduces': app['desiredReduces'] or 0,
          'durationFormatted': app['durationFormatted'],
          'startTimeFormatted': app['startTimeFormatted'],
          'diagnostics': app['diagnostics'] if app['diagnostics'] else '',
          'tasks': [],
          'metadata': [],
          'counters': []
      }
    elif app['applicationType'] == 'SPARK':
      app['logs'] = job.logs_url if hasattr(job, 'logs_url') else ''
      app['trackingUrl'] = job.trackingUrl if hasattr(job, 'trackingUrl') else ''
      common['type'] = 'SPARK'
      common['properties'] = {
        'metadata': [{'name': name, 'value': value} for name, value in app.items() if name != "url" and name != "killUrl"],
        'executors': []
      }
      if hasattr(job, 'metrics'):
        common['metrics'] = job.metrics
    elif app['applicationType'] == 'YarnV2':
      common['applicationType'] = app.get('type')
      common['properties'] = {
        'startTime': job.startTime,
        'finishTime': job.finishTime,
        'elapsedTime': job.duration,
        'attempts': [],
        'diagnostics': job.diagnostics
      }

    return common
Example #16
0
 def kill(self, job_id):
     app_id = job_id.replace('job', 'application')
     get_resource_manager().kill(app_id)  # We need to call the RM
Example #17
0
 def kill(self, job_id):
   app_id = job_id.replace('job', 'application')
   get_resource_manager().kill(app_id) # We need to call the RM
Example #18
0
 def __init__(self, user):
   self.user = user
   self.resource_manager_api = resource_manager_api.get_resource_manager()
   self.mapreduce_api = mapreduce_api.get_mapreduce_api()
   self.node_manager_api = node_manager_api.get_resource_manager_api()
   self.history_server_api = history_server_api.get_history_server_api()
Example #19
0
File: api.py Project: yhanwen/hue
 def __init__(self, user):
     self.user = user
     self.resource_manager_api = resource_manager_api.get_resource_manager()
     self.mapreduce_api = mapreduce_api.get_mapreduce_api()
     self.node_manager_api = node_manager_api.get_resource_manager_api()
     self.history_server_api = history_server_api.get_history_server_api()