def get_spark_status(user):
from spark.livy_client import get_api
status = None
try:
if not 'test' in sys.argv: # Avoid tests hanging
get_api(user).get_status()
status = 'OK'
except:
LOG.exception('failed to get spark status')
return status
def profile(self, appid, app_type, app_property, app_filters):
appid = appid.rsplit('-')[-1]
if app_property == 'properties':
api = get_api(self.user)
return api.get_statements(appid)
else:
return {}
def apps(self, filters):
api = get_api(self.user)
jobs = api.get_sessions()
return {
'apps': [{
'id': 'livy-%(id)s' % app,
'name': '%(kind)s %(id)s' % app,
'status': app['state'],
'apiStatus': self._api_status(app['state']),
'type': 'livy-session',
'user': app['owner'],
'progress': 100,
'queue': 'group',
'duration': 1,
'submitted': ''
} for app in jobs['sessions']],
'total': jobs['total']
}
def app(self, appid):
appid = appid.rsplit('-')[-1]
api = get_api(self.user)
job = api.get_session(appid)
return {
'id': 'livy-%(id)s' % job,
'name': '%(kind)s %(id)s' % job,
'status': job['state'],
'apiStatus': self._api_status(job['state']),
'type': 'livy-session',
'user': job['owner'],
'progress': 100,
'queue': 'group',
'duration': 1,
'submitted': '',
'properties': {
'statements': []
}
}
def apps(self, filters):
kwargs = {}
api = get_api(self.user)
jobs = api.list_jobs(**kwargs)
return {
'apps': [{
'id': app['jobId'],
'name': app['creationDate'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'type': app['jobType'],
'user': '',
'progress': 100,
'duration': 10 * 3600,
'submitted': app['creationDate']
} for app in jobs['jobs']],
'total': len(jobs)
}