def get_spark_status(user):
from spark.job_server_api import get_api
status = None
try:
if not 'test' in sys.argv: # Avoid tests hanging
get_api(user).get_status()
status = 'OK'
except:
LOG.exception('failed to get spark status')
return status
def get_spark_status(user):
from spark.job_server_api import get_api
status = None
try:
if not 'test' in sys.argv: # Avoid tests hanging
get_api(user).get_status()
status = 'OK'
except:
LOG.exception('failed to get spark status')
return status
def get_spark_status(user):
from spark.job_server_api import get_api
status = None
try:
if not 'test' in sys.argv: # Avoid tests hanging
get_api(user).get_status()
status = 'OK'
except:
pass
return status
def get_spark_status(user):
from spark.job_server_api import get_api
status = None
try:
if not 'test' in sys.argv: # Avoid tests hanging
get_api(user).get_status()
status = 'OK'
except:
pass
return status
def job(request, job_id):
api = get_api(request.user)
response = {}
try:
response['results'] = api.job(job_id)
except RestException, e:
response['results'] = json.loads(e.message)
def job(request, job_id):
api = get_api(request.user)
response = {}
try:
response['results'] = api.job(job_id)
except RestException, e:
response['results'] = json.loads(e.message)
def save_query(request, design_id=None):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
form = QueryForm()
api = get_api(request.user)
app_names = api.jars()
try:
form.bind(request.POST)
form.query.fields['appName'].choices = ((key, key)
for key in app_names)
if form.is_valid():
design = save_design(request, form, query_type, design, True)
response['design_id'] = design.id
response['status'] = 0
else:
response['message'] = smart_str(form.query.errors) + smart_str(
form.saveform.errors)
except RuntimeError, e:
response['message'] = str(e)
def save_query(request, design_id=None):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
form = QueryForm()
api = get_api(request.user)
app_names = api.jars()
try:
form.bind(request.POST)
form.query.fields['appName'].choices = ((key, key) for key in app_names)
if form.is_valid():
design = save_design(request, form, query_type, design, True)
response['design_id'] = design.id
response['status'] = 0
else:
response['message'] = smart_str(form.query.errors) + smart_str(form.saveform.errors)
except RuntimeError, e:
response['message'] = str(e)
def contexts(request):
api = get_api(request.user)
response = {
'contexts': api.contexts()
}
return HttpResponse(json.dumps(response), mimetype="application/json")
def list_jobs(request):
api = get_api(request.user)
jobs = api.jobs()
return render('list_jobs.mako', request, {
'jobs': jobs,
'jobs_json': json.dumps(jobs)
})
def list_applications(request):
api = get_api(request.user)
applications = api.jars()
return render('list_applications.mako', request, {
'applications': applications,
'applications_json': json.dumps([applications])
})
def list_contexts(request):
api = get_api(request.user)
contexts = api.contexts()
return render('list_contexts.mako', request, {
'contexts': contexts,
'contexts_json': json.dumps(contexts)
})
def list_jobs(request):
api = get_api(request.user)
jobs = api.jobs()
return render('list_jobs.mako', request, {
'jobs': jobs,
'jobs_json': json.dumps(jobs)
})
def job(request, job_id):
api = get_api(request.user)
response = {
'results': api.job(job_id)
}
return HttpResponse(json.dumps(response), mimetype="application/json")
def list_contexts(request):
api = get_api(request.user)
contexts = api.contexts()
return render('list_contexts.mako', request, {
'contexts': contexts,
'contexts_json': json.dumps(contexts)
})
def delete_contexts(request):
if request.method == 'POST':
api = get_api(request.user)
ids = request.POST.getlist('contexts_selection')
for name in ids:
api.delete_context(name)
return redirect(reverse('spark:list_contexts'))
else:
return render('confirm.mako', request, {'url': request.path, 'title': _('Delete context(s)?')})
def profile(self, appid, app_type, app_property, app_filters):
appid = appid.rsplit('-')[-1]
if app_property == 'properties':
api = get_api(self.user)
return api.get_statements(appid)
else:
return {}
def list_applications(request):
api = get_api(request.user)
applications = api.jars()
return render(
'list_applications.mako', request, {
'applications': applications,
'applications_json': json.dumps([applications])
})
def delete_contexts(request):
if request.method == "POST":
api = get_api(request.user)
ids = request.POST.getlist("contexts_selection")
for name in ids:
api.delete_context(name)
return redirect(reverse("spark:list_contexts"))
else:
return render("confirm.mako", request, {"url": request.path, "title": _("Delete context(s)?")})
def list_applications(request):
api = get_api(request.user)
applications = api.jars()
return render(
"list_applications.mako",
request,
{"applications": applications, "applications_json": json.dumps([applications])},
)
def profile(self, appid, app_type, app_property, app_filters):
appid = appid.rsplit('-')[-1]
if app_property == 'properties':
api = get_api(self.user)
return api.get_statements(appid)
else:
return {}
def get_query_form(request):
api = get_api(request.user)
app_names = api.jars()
if not app_names:
raise RuntimeError(_("Missing application jar list."))
form = SparkForm(request.POST, app_names=app_names)
return form
def download_result(request, job_id):
api = get_api(request.user)
result = api.job(job_id)
mimetype = "application/json"
gen = json.dumps(result["result"])
resp = HttpResponse(gen, mimetype=mimetype)
resp["Content-Disposition"] = "attachment; filename=query_result.%s" % format
return resp
def get_query_form(request):
api = get_api(request.user)
app_names = api.jars()
if not app_names:
raise RuntimeError(_("Missing application jar list."))
form = SparkForm(request.POST, app_names=app_names)
return form
def delete_contexts(request):
if request.method == 'POST':
api = get_api(request.user)
ids = request.POST.getlist('contexts_selection')
for name in ids:
api.delete_context(name)
return redirect(reverse('spark:list_contexts'))
else:
return render('confirm.mako', request, {
'url': request.path,
'title': _('Delete context(s)?')
})
def download_result(request, job_id):
api = get_api(request.user)
result = api.job(job_id)
mimetype = 'application/json'
gen = json.dumps(result['result'])
resp = HttpResponse(gen, mimetype=mimetype)
resp[
'Content-Disposition'] = 'attachment; filename=query_result.%s' % format
return resp
def get_spark_status(user):
from spark.job_server_api import get_api
status = None
try:
if not 'test' in sys.argv: # Avoid tests hanging
status = str(get_api(user).get_status())
except ValueError:
# No json returned
status = 'OK'
except:
pass
return status
def get_spark_status(user):
from spark.job_server_api import get_api
status = None
try:
if not 'test' in sys.argv: # Avoid tests hanging
status = str(get_api(user).get_status())
except ValueError:
# No json returned
status = 'OK'
except:
pass
return status
def execute(request, design_id=None):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
try:
form = get_query_form(request)
if form.is_valid():
#design = save_design(request, SaveForm(), form, query_type, design)
# query = SQLdesign(form, query_type=query_type)
# query_server = dbms.get_query_server_config(request.POST.get('server'))
# db = dbms.get(request.user, query_server)
# query_history = db.execute_query(query, design)
# query_history.last_state = beeswax_models.QueryHistory.STATE.expired.index
# query_history.save()
params = '\n'.join([
'%(name)s=%(value)s' % param
for param in json.loads(form.cleaned_data['params'])
])
try:
api = get_api(request.user)
results = api.submit_job(
form.cleaned_data['appName'],
form.cleaned_data['classPath'],
data=params,
context=None if form.cleaned_data['autoContext'] else
form.cleaned_data['context'],
sync=False)
if results['status'] == 'STARTED':
response['status'] = 0
response['results'] = results
else:
response['message'] = str(results[1]['result'])
response['design'] = design.id
except Exception, e:
response['message'] = str(e)
else:
def delete_context(request):
if request.method != 'DELETE':
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Requires a DELETE'))
response = {}
name = request.POST.get('name', '')
api = get_api(request.user)
try:
response = api.delete_context(name)
except ValueError:
# No json is returned
response = {'status': 'OK'}
except Exception, e:
response = json.loads(e.message)
def delete_context(request):
if request.method != 'DELETE':
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Requires a DELETE'))
response = {}
name = request.POST.get('name', '')
api = get_api(request.user)
try:
response = api.delete_context(name)
except ValueError:
# No json is returned
response = {'status': 'OK'}
except Exception, e:
response = json.loads(e.message)
def execute(request, design_id=None):
response = {'status': -1, 'message': ''}
if request.method != 'POST':
response['message'] = _('A POST request is required.')
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
try:
form = get_query_form(request)
if form.is_valid():
#design = save_design(request, SaveForm(), form, query_type, design)
# query = SQLdesign(form, query_type=query_type)
# query_server = dbms.get_query_server_config(request.POST.get('server'))
# db = dbms.get(request.user, query_server)
# query_history = db.execute_query(query, design)
# query_history.last_state = beeswax_models.QueryHistory.STATE.expired.index
# query_history.save()
params = '\n'.join(['%(name)s=%(value)s' % param for param in json.loads(form.cleaned_data['params'])])
try:
api = get_api(request.user)
results = api.submit_job(
form.cleaned_data['appName'],
form.cleaned_data['classPath'],
data=params,
context=None if form.cleaned_data['autoContext'] else form.cleaned_data['context'],
sync=False
)
if results['status'] == 'STARTED':
response['status'] = 0
response['results'] = results
else:
response['message'] = str(results[1]['result'])
response['design'] = design.id
except Exception, e:
response['message'] = str(e)
else:
def create_context(request):
if request.method != 'POST':
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Requires a POST'))
response = {}
name = request.POST.get('name', '')
memPerNode = request.POST.get('mem-per-node', '512m')
numCores = request.POST.get('num-cpu-cores', '1')
api = get_api(request.user)
try:
response = api.create_context(name, memPerNode=memPerNode, numCores=numCores)
except Exception, e:
if e.message != 'No JSON object could be decoded':
response = json.loads(e.message)
else:
response = {'status': 'OK'}
def editor(request, design_id=None, query_history_id=None):
api = get_api(request.user)
jobs = api.jobs()
if design_id is not None and not design_id.isdigit():
job_id, design_id = design_id, None
else:
job_id = None
action = request.path
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
return render('editor.mako', request, {
'action': action,
'design': design,
'design_json': json.dumps(design_to_dict(design)),
'can_edit_name': design.id and not design.is_auto,
'job_id': job_id,
})
def apps(self, filters):
api = get_api(self.user)
jobs = api.get_sessions()
return {
'apps': [{
'id': 'livy-%(id)s' % app,
'name': '%(kind)s %(id)s' % app,
'status': app['state'],
'apiStatus': self._api_status(app['state']),
'type': 'livy-session',
'user': app['owner'],
'progress': 100,
'queue': 'group',
'duration': 1,
'submitted': ''
} for app in jobs['sessions']],
'total': jobs['total']
}
def create_context(request):
if request.method != 'POST':
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Requires a POST'))
response = {}
name = request.POST.get('name', '')
memPerNode = request.POST.get('mem-per-node', '512m')
numCores = request.POST.get('num-cpu-cores', '1')
api = get_api(request.user)
try:
response = api.create_context(name,
memPerNode=memPerNode,
numCores=numCores)
except ValueError:
# No json is returned
response = {'status': 'OK'}
except Exception, e:
response = json.loads(e.message)
def upload_app(request):
if request.method != "POST":
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_("Requires a POST"))
response = {"status": -1}
form = UploadApp(request.POST, request.FILES)
if form.is_valid():
app_name = form.cleaned_data["app_name"]
try:
data = form.cleaned_data["jar_file"].read()
api = get_api(request.user)
response["status"] = 0
response["results"] = api.upload_jar(app_name, data)
except ValueError:
# No json is returned
pass
else:
response["results"] = form.errors
return redirect(request.META["HTTP_REFERER"])
def editor(request, design_id=None, query_history_id=None):
api = get_api(request.user)
jobs = api.jobs()
if design_id is not None and not design_id.isdigit():
job_id, design_id = design_id, None
else:
job_id = None
action = request.path
app_name = get_app_name(request)
query_type = beeswax_models.SavedQuery.TYPES_MAPPING[app_name]
design = safe_get_design(request, query_type, design_id)
return render(
'editor.mako', request, {
'action': action,
'design': design,
'design_json': json.dumps(design_to_dict(design)),
'can_edit_name': design.id and not design.is_auto,
'job_id': job_id,
})
def apps(self, filters):
kwargs = {}
api = get_api(self.user)
jobs = api.list_jobs(**kwargs)
return {
'apps': [{
'id': app['jobId'],
'name': app['creationDate'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'type': app['jobType'],
'user': '',
'progress': 100,
'duration': 10 * 3600,
'submitted': app['creationDate']
} for app in jobs['jobs']],
'total': len(jobs)
}
def apps(self, filters):
api = get_api(self.user)
jobs = api.get_sessions()
return {
'apps': [{
'id': 'livy-%(id)s' % app,
'name': '%(kind)s %(id)s' % app,
'status': app['state'],
'apiStatus': self._api_status(app['state']),
'type': 'livy-session',
'user': app['owner'],
'progress': 100,
'queue': 'group',
'duration': 1,
'submitted': ''
} for app in jobs['sessions']],
'total':
jobs['total']
}
def app(self, appid):
appid = appid.rsplit('-')[-1]
api = get_api(self.user)
job = api.get_session(appid)
return {
'id': 'livy-%(id)s' % job,
'name': '%(kind)s %(id)s' % job,
'status': job['state'],
'apiStatus': self._api_status(job['state']),
'type': 'livy-session',
'user': job['owner'],
'progress': 100,
'queue': 'group',
'duration': 1,
'submitted': '',
'properties': {
'statements': []
}
}
def app(self, appid):
appid = appid.rsplit('-')[-1]
api = get_api(self.user)
job = api.get_session(appid)
return {
'id': 'livy-%(id)s' % job,
'name': '%(kind)s %(id)s' % job,
'status': job['state'],
'apiStatus': self._api_status(job['state']),
'type': 'livy-session',
'user': job['owner'],
'progress': 100,
'queue': 'group',
'duration': 1,
'submitted': '',
'properties': {
'statements': []
}
}
def apps(self, filters):
kwargs = {}
api = get_api(self.user)
jobs = api.list_jobs(**kwargs)
return {
'apps': [{
'id': app['jobId'],
'name': app['creationDate'],
'status': app['status'],
'apiStatus': self._api_status(app['status']),
'type': app['jobType'],
'user': '',
'progress': 100,
'duration': 10 * 3600,
'submitted': app['creationDate']
} for app in jobs['jobs']],
'total':
len(jobs)
}
def upload_app(request):
if request.method != 'POST':
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Requires a POST'))
response = {'status': -1}
form = UploadApp(request.POST, request.FILES)
if form.is_valid():
app_name = form.cleaned_data['app_name']
try:
data = form.cleaned_data['jar_file'].read()
api = get_api(request.user)
response['status'] = 0
response['results'] = api.upload_jar(app_name, data)
except ValueError:
# No json is returned
pass
else:
response['results'] = form.errors
return redirect(request.META['HTTP_REFERER'])
def upload_app(request):
if request.method != 'POST':
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Requires a POST'))
response = {
'status': -1
}
form = UploadApp(request.POST, request.FILES)
if form.is_valid():
app_name = form.cleaned_data['app_name']
try:
data = form.cleaned_data['jar_file'].read()
api = get_api(request.user)
response['status'] = 0
response['results'] = api.upload_jar(app_name, data)
except ValueError:
# No json is returned
pass
else:
response['results'] = form.errors
return redirect(request.META['HTTP_REFERER'])
def contexts(request):
api = get_api(request.user)
response = {'contexts': api.contexts()}
return HttpResponse(json.dumps(response), mimetype="application/json")
def list_jobs(request):
api = get_api(request.user)
jobs = api.jobs()
return render("list_jobs.mako", request, {"jobs": jobs, "jobs_json": json.dumps(jobs)})
def list_contexts(request):
api = get_api(request.user)
contexts = api.contexts()
return render("list_contexts.mako", request, {"contexts": contexts, "contexts_json": json.dumps(contexts)})