def __init__(self, user):
self.user = user
self.resource_manager_api = resource_manager_api.get_resource_manager(
user.username)
self.mapreduce_api = mapreduce_api.get_mapreduce_api(user.username)
self.history_server_api = history_server_api.get_history_server_api()
self.spark_history_server_api = spark_history_server_api.get_history_server_api(
)
def app(self, appid):
try:
job = NativeYarnApi(self.user).get_job(jobid=appid)
except ApplicationNotRunning, e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(self.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
def get_job(request, job_id):
try:
job = get_api(request.user, request.jt).get_job(jobid=job_id)
except ApplicationNotRunning, e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(request.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
def get_job(request, job_id):
try:
job = get_api(request.user, request.jt).get_job(jobid=job_id)
except ApplicationNotRunning, e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(request.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
def app(self, appid):
try:
job = NativeYarnApi(self.user).get_job(jobid=appid)
except ApplicationNotRunning, e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(self.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
def __init__(self, user):
self.user = user
self.resource_manager_api = None
self.mapreduce_api = None
self.history_server_api = None
self.spark_history_server_api = None
if list(YARN_CLUSTERS.keys()):
self.resource_manager_api = resource_manager_api.get_resource_manager(user.username)
self.mapreduce_api = mapreduce_api.get_mapreduce_api(user.username)
self.history_server_api = history_server_api.get_history_server_api(user.username)
self.spark_history_server_api = spark_history_server_api.get_history_server_api() # Spark HS does not support setuser
def decorate(request, *args, **kwargs):
jobid = kwargs['job']
try:
job = get_api(request.user, request.jt).get_job(jobid=jobid)
except ApplicationNotRunning, e:
if e.job.get('state', '').lower() == 'accepted' and 'kill' in request.path:
rm_api = resource_manager_api.get_resource_manager(request.user)
job = Application(e.job, rm_api)
else:
# reverse() seems broken, using request.path but beware, it discards GET and POST info
return job_not_assigned(request, jobid, request.path)
def decorate(request, *args, **kwargs):
jobid = kwargs['job']
try:
job = get_api(request.user, request.jt).get_job(jobid=jobid)
except ApplicationNotRunning, e:
if e.job.get('state', '').lower() == 'accepted' and 'kill' in request.path:
rm_api = resource_manager_api.get_resource_manager(request.user)
job = Application(e.job, rm_api)
else:
# reverse() seems broken, using request.path but beware, it discards GET and POST info
return job_not_assigned(request, jobid, request.path)
def test_yarn_configurations():
# Single cluster for now
result = []
try:
url = ''
api = get_resource_manager()
url = api._url
api.apps()
except Exception, e:
msg = 'Failed to contact Resource Manager at %s: %s' % (url, e)
result.append(('Resource Manager', msg))
def test_yarn_configurations():
# Single cluster for now
result = []
try:
url = ''
api = get_resource_manager()
url = api._url
api.apps()
except Exception, e:
msg = 'Failed to contact Resource Manager at %s: %s' % (url, e)
result.append(('Resource Manager', msg))
def test_yarn_configurations(user):
# Single cluster for now
from hadoop.yarn.resource_manager_api import get_resource_manager
result = []
try:
url = ''
api = get_resource_manager(user.username)
url = api._url
api.apps()
except Exception, e:
msg = 'Failed to contact Resource Manager at %s: %s' % (url, e)
LOG.exception(msg)
result.append(('Resource Manager', msg))
def test_yarn_configurations(user):
# Single cluster for now
from hadoop.yarn.resource_manager_api import get_resource_manager
result = []
try:
url = ''
api = get_resource_manager(user.username)
url = api._url
api.apps()
except Exception, e:
msg = 'Failed to contact Resource Manager at %s: %s' % (url, e)
LOG.exception(msg)
result.append(('Resource Manager', msg))
def get_job(request, job_id):
try:
job = get_api(request.user, request.jt).get_job(jobid=job_id)
except ApplicationNotRunning as e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(request.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
except JobExpired as e:
raise PopupException(
_('Job %s has expired.') % job_id,
detail=_('Cannot be found on the History Server.'))
except Exception as e:
msg = 'Could not find job %s.'
LOG.exception(msg % job_id)
raise PopupException(_(msg) % job_id, detail=e)
return job
def __init__(self, user): self.user = user self.resource_manager_api = resource_manager_api.get_resource_manager(user.username) self.mapreduce_api = mapreduce_api.get_mapreduce_api(user.username) self.history_server_api = history_server_api.get_history_server_api(user.username) self.spark_history_server_api = spark_history_server_api.get_history_server_api()
def app(self, appid):
try:
job = NativeYarnApi(self.user).get_job(jobid=appid)
except ApplicationNotRunning as e:
if e.job.get('state', '').lower() == 'accepted':
rm_api = resource_manager_api.get_resource_manager(self.user)
job = Application(e.job, rm_api)
else:
raise e # Job has not yet been accepted by RM
except JobExpired as e:
raise PopupException(_('Job %s has expired.') % appid, detail=_('Cannot be found on the History Server.'))
except Exception as e:
msg = 'Could not find job %s.'
LOG.exception(msg % appid)
raise PopupException(_(msg) % appid, detail=e)
app = massage_job_for_json(job, user=self.user)
common = {
'id': app['id'],
'name': app['name'],
'type': app['applicationType'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'user': app['user'],
'progress': app['progress'],
'duration': app['durationMs'],
'submitted': app['startTimeMs'],
'canWrite': app['canKill'],
}
if app['applicationType'] == 'MR2' or app['applicationType'] == 'MAPREDUCE':
common['type'] = 'MAPREDUCE'
if app['desiredMaps'] is None or app['finishedMaps'] is None:
app['mapsPercentComplete'] = 100
if app['desiredReduces'] is None or app['finishedReduces'] is None:
app['reducesPercentComplete'] = 100
common['properties'] = {
'maps_percent_complete': app['mapsPercentComplete'] or 0,
'reduces_percent_complete': app['reducesPercentComplete'] or 0,
'finishedMaps': app['finishedMaps'] or 0,
'finishedReduces': app['finishedReduces'] or 0,
'desiredMaps': app['desiredMaps'] or 0,
'desiredReduces': app['desiredReduces'] or 0,
'durationFormatted': app['durationFormatted'],
'startTimeFormatted': app['startTimeFormatted'],
'diagnostics': app['diagnostics'] if app['diagnostics'] else '',
'tasks': [],
'metadata': [],
'counters': []
}
elif app['applicationType'] == 'SPARK':
app['logs'] = job.logs_url if hasattr(job, 'logs_url') else ''
app['trackingUrl'] = job.trackingUrl if hasattr(job, 'trackingUrl') else ''
common['type'] = 'SPARK'
common['properties'] = {
'metadata': [{'name': name, 'value': value} for name, value in app.items() if name != "url" and name != "killUrl"],
'executors': []
}
if hasattr(job, 'metrics'):
common['metrics'] = job.metrics
elif app['applicationType'] == 'YarnV2':
common['applicationType'] = app.get('type')
common['properties'] = {
'startTime': job.startTime,
'finishTime': job.finishTime,
'elapsedTime': job.duration,
'attempts': [],
'diagnostics': job.diagnostics
}
return common
def kill(self, job_id):
app_id = job_id.replace('job', 'application')
get_resource_manager().kill(app_id) # We need to call the RM
def kill(self, job_id):
app_id = job_id.replace('job', 'application')
get_resource_manager().kill(app_id) # We need to call the RM
def __init__(self, user): self.user = user self.resource_manager_api = resource_manager_api.get_resource_manager() self.mapreduce_api = mapreduce_api.get_mapreduce_api() self.node_manager_api = node_manager_api.get_resource_manager_api() self.history_server_api = history_server_api.get_history_server_api()
def __init__(self, user):
self.user = user
self.resource_manager_api = resource_manager_api.get_resource_manager()
self.mapreduce_api = mapreduce_api.get_mapreduce_api()
self.node_manager_api = node_manager_api.get_resource_manager_api()
self.history_server_api = history_server_api.get_history_server_api()
