def stop(request):
if request.method != "POST":
raise PopupException(_("POST request required."))
spark_script = SparkScript.objects.get(id=request.POST.get("id"))
job_id = spark_script.dict["job_id"]
job = check_job_access_permission(request, job_id)
check_job_edition_permission(job, request.user)
try:
api.get(request.fs, request.jt, request.user).stop(job_id)
except RestException, e:
raise PopupException(_("Error stopping Pig script.") % e.message)
def stop(request):
if request.method != 'POST':
raise PopupException(_('POST request required.'))
spark_script = SparkScript.objects.get(id=request.POST.get('id'))
job_id = spark_script.dict['job_id']
job = check_job_access_permission(request, job_id)
check_job_edition_permission(job, request.user)
try:
api.get(request.fs, request.jt, request.user).stop(job_id)
except RestException, e:
raise PopupException(_("Error stopping Pig script.") % e.message)
def run(request):
if request.method != "POST":
raise PopupException(_("POST request required."))
attrs = {
"id": request.POST.get("id"),
"name": request.POST.get("name"),
"script": request.POST.get("script"),
"user": request.user,
"parameters": json.loads(request.POST.get("parameters")),
"resources": json.loads(request.POST.get("resources")),
"hadoopProperties": json.loads(request.POST.get("hadoopProperties")),
"language": json.loads(request.POST.get("language")),
"is_design": False,
}
spark_script = create_or_update_script(**attrs)
params = request.POST.get("submissionVariables")
oozie_id = api.get(request.fs, request.jt, request.user).submit(spark_script, params)
spark_script.update_from_dict({"job_id": oozie_id})
spark_script.save()
response = {
"id": spark_script.id,
"watchUrl": reverse("spark:watch", kwargs={"job_id": oozie_id}) + "?format=python",
}
return HttpResponse(json.dumps(response), content_type="text/plain")
def run(request):
if request.method != 'POST':
raise PopupException(_('POST request required.'))
attrs = {
'id': request.POST.get('id'),
'name': request.POST.get('name'),
'script': request.POST.get('script'),
'user': request.user,
'parameters': json.loads(request.POST.get('parameters')),
'resources': json.loads(request.POST.get('resources')),
'hadoopProperties': json.loads(request.POST.get('hadoopProperties')),
'is_design': False
}
spark_script = create_or_update_script(**attrs)
params = request.POST.get('submissionVariables')
oozie_id = api.get(request.fs, request.jt, request.user).submit(spark_script, params)
spark_script.update_from_dict({'job_id': oozie_id})
spark_script.save()
response = {
'id': spark_script.id,
'watchUrl': reverse('spark:watch', kwargs={'job_id': oozie_id}) + '?format=python'
}
return HttpResponse(json.dumps(response), content_type="text/plain")
def run(request):
if request.method != 'POST':
raise PopupException(_('POST request required.'))
attrs = {
'id': request.POST.get('id'),
'name': request.POST.get('name'),
'script': request.POST.get('script'),
'user': request.user,
'parameters': json.loads(request.POST.get('parameters')),
'resources': json.loads(request.POST.get('resources')),
'hadoopProperties': json.loads(request.POST.get('hadoopProperties')),
'language': json.loads(request.POST.get('language')),
'is_design': False
}
spark_script = create_or_update_script(**attrs)
params = request.POST.get('submissionVariables')
oozie_id = api.get(request.fs, request.jt,
request.user).submit(spark_script, params)
spark_script.update_from_dict({'job_id': oozie_id})
spark_script.save()
response = {
'id':
spark_script.id,
'watchUrl':
reverse('spark:watch', kwargs={'job_id': oozie_id}) + '?format=python'
}
return HttpResponse(json.dumps(response), content_type="text/plain")
def dashboard(request):
spark_api = api.get(request.fs, request.jt, request.user)
jobs = spark_api.get_jobs()
hue_jobs = Document.objects.available(SparkScript, request.user)
massaged_jobs = spark_api.massaged_jobs_for_json(request, jobs, hue_jobs)
return HttpResponse(json.dumps(massaged_jobs), mimetype="application/json")
def dashboard(request):
spark_api = api.get(request.fs, request.jt, request.user)
jobs = spark_api.get_jobs()
hue_jobs = Document.objects.available(SparkScript, request.user)
massaged_jobs = spark_api.massaged_jobs_for_json(request, jobs, hue_jobs)
return HttpResponse(json.dumps(massaged_jobs), mimetype="application/json")
def watch(request, job_id):
oozie_workflow = check_job_access_permission(request, job_id)
logs, workflow_actions = api.get(request.jt, request.jt, request.user).get_log(request, oozie_workflow)
output = get_workflow_output(oozie_workflow, request.fs)
workflow = {
"job_id": oozie_workflow.id,
"status": oozie_workflow.status,
"progress": oozie_workflow.get_progress(),
"isRunning": oozie_workflow.is_running(),
"killUrl": reverse("oozie:manage_oozie_jobs", kwargs={"job_id": oozie_workflow.id, "action": "kill"}),
"rerunUrl": reverse(
"oozie:rerun_oozie_job", kwargs={"job_id": oozie_workflow.id, "app_path": oozie_workflow.appPath}
),
"actions": workflow_actions,
}
response = {"workflow": workflow, "logs": logs, "output": hdfs_link(output)}
return HttpResponse(json.dumps(response), content_type="text/plain")
def watch(request, job_id):
oozie_workflow = check_job_access_permission(request, job_id)
logs, workflow_actions = api.get(request.jt, request.jt, request.user).get_log(request, oozie_workflow)
output = get_workflow_output(oozie_workflow, request.fs)
workflow = {
'job_id': oozie_workflow.id,
'status': oozie_workflow.status,
'progress': oozie_workflow.get_progress(),
'isRunning': oozie_workflow.is_running(),
'killUrl': reverse('oozie:manage_oozie_jobs', kwargs={'job_id': oozie_workflow.id, 'action': 'kill'}),
'rerunUrl': reverse('oozie:rerun_oozie_job', kwargs={'job_id': oozie_workflow.id, 'app_path': oozie_workflow.appPath}),
'actions': workflow_actions
}
response = {
'workflow': workflow,
'logs': logs,
'output': hdfs_link(output)
}
return HttpResponse(json.dumps(response), content_type="text/plain")
def watch(request, job_id):
oozie_workflow = check_job_access_permission(request, job_id)
logs, workflow_actions = api.get(request.jt, request.jt,
request.user).get_log(
request, oozie_workflow)
output = get_workflow_output(oozie_workflow, request.fs)
workflow = {
'job_id':
oozie_workflow.id,
'status':
oozie_workflow.status,
'progress':
oozie_workflow.get_progress(),
'isRunning':
oozie_workflow.is_running(),
'killUrl':
reverse('oozie:manage_oozie_jobs',
kwargs={
'job_id': oozie_workflow.id,
'action': 'kill'
}),
'rerunUrl':
reverse('oozie:rerun_oozie_job',
kwargs={
'job_id': oozie_workflow.id,
'app_path': oozie_workflow.appPath
}),
'actions':
workflow_actions
}
response = {
'workflow': workflow,
'logs': logs,
'output': hdfs_link(output)
}
return HttpResponse(json.dumps(response), content_type="text/plain")
def test_parse_oozie_logs(self):
api = get(None, None, self.user)
assert_equal('''Stdoutput aaa''',
api._match_logs({'logs': [None, OOZIE_LOGS]}))
def test_parse_oozie_logs(self):
api = get(None, None, self.user)
assert_equal('''Stdoutput aaa''', api._match_logs({'logs': [None, OOZIE_LOGS]}))
