def new_design(request, node_type):
"""
Designs are the interpolation of Workflows and a single action.
Save ``name`` and ``description`` of workflows.
Also, use ``id`` of workflows.
"""
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405)
workflow = Workflow.objects.new_workflow(request.user)
ActionForm = design_form_by_type(node_type, request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)
workflow.managed = False
# Every one should be able to execute and clone a design.
workflow.is_shared = True
workflow.save()
Workflow.objects.initialize(workflow, request.fs)
action = form.save(commit=False)
action.workflow = workflow
action.node_type = node_type
action.save()
workflow.start.add_node(action)
action.add_node(workflow.end)
workflow.name = request.POST.get('name')
workflow.description = request.POST.get('description')
workflow.save()
data = format_dict_field_values(request.POST.copy())
data['id'] = workflow.id
return render_json(data)
def workflow_validate_node_json(node_type, node_dict, errors, user, workflow):
"""
Validates a single node.
node_type is the node type of the action information passed.
node_dict is a dictionary describing the node.
errors is a dictionary that will be populated with any found errors.
Returns Boolean.
"""
assert isinstance(errors, dict), "errors must be a dict."
if node_type in ACTION_TYPES:
form_class = design_form_by_type(node_type, user, workflow)
else:
form_class = NodeForm
form = form_class(data=node_dict)
if form.is_valid():
for field in form.fields:
errors[field] = []
return True
else:
for field in form.fields:
errors[field] = form[field].errors
return False
def save_design(request, design_id):
workflow = _get_design(design_id)
_check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id})
ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)
data = format_dict_field_values(request.POST.copy())
sanitize_node_dict(data)
workflow.name = data['name']
workflow.description = data['description']
node = workflow.start.get_child('to').get_full_node()
node_id = node.id
for key in data:
setattr(node, key, data[key])
node.id = node_id
node.pk = node_id
node.save()
workflow.save()
data['id'] = workflow.id
return render_json(data);
def edit_workflow(request, workflow):
history = History.objects.filter(submitter=request.user,
job=workflow).order_by('-submission_date')
workflow_form = WorkflowForm(instance=workflow)
user_can_access_job = workflow.is_accessible(request.user)
user_can_edit_job = workflow.is_editable(request.user)
return render(
'editor/edit_workflow.mako', request, {
'workflow_form':
workflow_form,
'workflow':
workflow,
'history':
history,
'user_can_access_job':
user_can_access_job,
'user_can_edit_job':
user_can_edit_job,
'job_properties':
extract_field_data(workflow_form['job_properties']),
'link_form':
LinkForm(),
'default_link_form':
DefaultLinkForm(action=workflow.start),
'node_form':
NodeForm(),
'action_forms':
[(node_type, design_form_by_type(node_type, request.user,
workflow)())
for node_type in ACTION_TYPES.iterkeys()]
})
def edit_workflow(request, workflow):
history = History.objects.filter(submitter=request.user, job=workflow).order_by("-submission_date")
workflow_form = WorkflowForm(instance=workflow)
user_can_access_job = workflow.is_accessible(request.user)
user_can_edit_job = workflow.is_editable(request.user)
return render(
"editor/edit_workflow.mako",
request,
{
"workflow_form": workflow_form,
"workflow": workflow,
"history": history,
"user_can_access_job": user_can_access_job,
"user_can_edit_job": user_can_edit_job,
"job_properties": extract_field_data(workflow_form["job_properties"]),
"link_form": LinkForm(),
"default_link_form": DefaultLinkForm(action=workflow.start),
"node_form": NodeForm(),
"action_forms": [
(node_type, design_form_by_type(node_type, request.user, workflow)())
for node_type in ACTION_TYPES.iterkeys()
],
},
)
def new_action(request, workflow, node_type, parent_action_id):
ActionForm = design_form_by_type(node_type)
if request.method == 'POST':
action_form = ActionForm(request.POST)
if action_form.is_valid():
action = action_form.save(commit=False)
action.node_type = node_type
action.workflow = workflow
action.save()
workflow.add_action(action, parent_action_id)
return redirect(reverse('oozie:edit_workflow', kwargs={'workflow': workflow.id}))
else:
action_form = ActionForm()
return render('editor/edit_workflow_action.mako', request, {
'workflow': workflow,
'job_properties': 'job_properties' in action_form.fields and extract_field_data(action_form['job_properties']) or '[]',
'files': 'files' in action_form.fields and extract_field_data(action_form['files']) or '[]',
'archives': 'archives' in action_form.fields and extract_field_data(action_form['archives']) or '[]',
'params': 'params' in action_form.fields and extract_field_data(action_form['params']) or '[]',
'prepares': 'prepares' in action_form.fields and extract_field_data(action_form['prepares']) or '[]',
'action_form': action_form,
'node_type': node_type,
'properties_hint': _STD_PROPERTIES_JSON,
'form_url': reverse('oozie:new_action', kwargs={'workflow': workflow.id,
'node_type': node_type,
'parent_action_id': parent_action_id}),
'can_edit_action': True,
})
def _validate_node_json(node_type, node_dict, errors, user, workflow):
"""
Validates a single node excluding links.
node_type is the node type of the action information passed.
node_dict is a dictionary describing the node.
errors is a dictionary that will be populated with any found errors.
user is a User object that is associated with the node_type. Only needed for Subworkflow node.
workflow is the Workflow object associated with the node. Only needed for Subworkflow node.
Returns Boolean.
"""
assert isinstance(errors, dict), "errors must be a dict."
if node_type in ACTION_TYPES:
form_class = design_form_by_type(node_type, user, workflow)
else:
form_class = NodeForm
form = form_class(data=node_dict)
if form.is_valid():
for field in form.fields:
errors[field] = []
return True
else:
for field in form.fields:
errors[field] = form[field].errors
return False
def run_jar(jar_hdfs_path, java_args, main_class, wf_name=None, description=None, username='******'):
user, created = User.objects.get_or_create(username=username)
url = WEBHDFS_URL
fs_defaultfs = FS_DEFAULTFS
fs = webhdfs.WebHdfs(url, fs_defaultfs)
if not wf_name:
wf_name = jar_hdfs_path
if not description:
description = java_args
POST = {u'files': u'[]', u'is_shared': u'true',
u'args': java_args,
u'name': wf_name, u'parameters': u'[{"name":"oozie.use.system.libpath","value":"true"}]',
u'workflow': u'0', u'java_opts': u'', u'jar_path': jar_hdfs_path, u'job_properties': u'[]',
u'job_xml': u'', u'main_class': main_class, u'prepares': u'[]',
u'node_type': u'java', u'last_modified': u'0', u'archives': u'[]', u'owner': u'',
u'capture_output': u'false', u'id': u'', u'description': description}
node_type = 'java'
workflow = Workflow.objects.new_workflow(user)
ActionForm = design_form_by_type(node_type, user, workflow)
form = ActionForm(POST)
workflow.managed = False
# Every one should be able to execute and clone a design.
workflow.is_shared = True
workflow.save()
Workflow.objects.initialize(workflow, fs)
action = form.save(commit=False)
action.workflow = workflow
action.node_type = node_type
action.save()
workflow.start.add_node(action)
action.add_node(workflow.end)
# Action form validates name and description.
workflow.name = POST.get('name')
workflow.description = POST.get('description')
workflow.save()
#save_design
data = format_dict_field_values(POST.copy())
save_design_my(workflow.id, data)
# after saving workflow we run it
submission = Submission(user, workflow, fs)
mapping = {u'oozie.use.system.libpath': u'true'}
#__import__("ipdb").set_trace()
job_id = None
attempts = ATTEMPTS_NUM
while attempts > 0:
try:
job_id = submit_workflow_my(user, fs, workflow, mapping)
break
except PopupException, ex:
__import__("ipdb").set_trace()
print str(attempts) + ", " + str(ex)
attempts -= 1
def new_action(request, workflow, node_type, parent_action_id):
ActionForm = design_form_by_type(node_type)
if request.method == 'POST':
action_form = ActionForm(request.POST)
if action_form.is_valid():
action = action_form.save(commit=False)
action.node_type = node_type
action.workflow = workflow
action.save()
workflow.add_action(action, parent_action_id)
return redirect(
reverse('oozie:edit_workflow',
kwargs={'workflow': workflow.id}))
else:
action_form = ActionForm()
return render(
'editor/edit_workflow_action.mako', request, {
'workflow':
workflow,
'job_properties':
'job_properties' in action_form.fields
and extract_field_data(action_form['job_properties']) or '[]',
'files':
'files' in action_form.fields
and extract_field_data(action_form['files']) or '[]',
'archives':
'archives' in action_form.fields
and extract_field_data(action_form['archives']) or '[]',
'params':
'params' in action_form.fields
and extract_field_data(action_form['params']) or '[]',
'prepares':
'prepares' in action_form.fields
and extract_field_data(action_form['prepares']) or '[]',
'action_form':
action_form,
'node_type':
node_type,
'properties_hint':
_STD_PROPERTIES_JSON,
'form_url':
reverse('oozie:new_action',
kwargs={
'workflow': workflow.id,
'node_type': node_type,
'parent_action_id': parent_action_id
}),
'can_edit_action':
True,
})
def save_design(request, design_id):
workflow = _get_design(design_id)
_check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id})
ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)
data = format_dict_field_values(request.POST.copy())
_save_design(design_id, data)
return get_design(request, design_id);
def save_design(request, design_id):
workflow = _get_design(design_id)
_check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id})
ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)
data = format_dict_field_values(request.POST.copy())
_save_design(design_id, data)
return get_design(request, design_id);
def edit_workflow(request, workflow):
history = History.objects.filter(submitter=request.user, job=workflow).order_by('-submission_date')
workflow_form = WorkflowForm(instance=workflow)
user_can_edit_job = workflow.is_editable(request.user)
return render('editor/edit_workflow.mako', request, {
'workflow_form': workflow_form,
'workflow': workflow,
'history': history,
'user_can_edit_job': user_can_edit_job,
'job_properties': extract_field_data(workflow_form['job_properties']),
'link_form': LinkForm(),
'default_link_form': DefaultLinkForm(action=workflow.start),
'action_forms': [(node_type, design_form_by_type(node_type, request.user, workflow)())
for node_type in ACTION_TYPES.iterkeys()]
})
def new_design(request, node_type):
"""
Designs are the interpolation of Workflows and a single action.
Save ``name`` and ``description`` of workflows.
Also, use ``id`` of workflows.
"""
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be POST request.'),
error_code=405)
workflow = Workflow.objects.new_workflow(request.user)
ActionForm = design_form_by_type(node_type, request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving design'),
data={'errors': form.errors},
error_code=400)
workflow.managed = False
# Every one should be able to execute and clone a design.
workflow.is_shared = True
workflow.save()
Workflow.objects.initialize(workflow, request.fs)
action = form.save(commit=False)
action.workflow = workflow
action.node_type = node_type
action.save()
workflow.start.add_node(action)
action.add_node(workflow.end)
# Action form validates name and description.
workflow.name = request.POST.get('name')
workflow.description = request.POST.get('description')
workflow.save()
doc = workflow.doc.get()
doc.extra = 'jobsub'
doc.save()
# Save design again to update all fields.
data = format_dict_field_values(request.POST.copy())
_save_design(workflow.id, data)
return get_design(request, workflow.id)
def new_design(request, node_type):
"""
Designs are the interpolation of Workflows and a single action.
Save ``name`` and ``description`` of workflows.
Also, use ``id`` of workflows.
"""
if request.method != "POST":
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_("Must be POST request."), error_code=405)
workflow = Workflow.objects.new_workflow(request.user)
ActionForm = design_form_by_type(node_type, request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(
code="INVALID_REQUEST_ERROR", message=_("Error saving design"), data={"errors": form.errors}, error_code=400
)
workflow.managed = False
# Every one should be able to execute and clone a design.
workflow.is_shared = True
workflow.save()
Workflow.objects.initialize(workflow, request.fs)
action = form.save(commit=False)
action.workflow = workflow
action.node_type = node_type
action.save()
workflow.start.add_node(action)
action.add_node(workflow.end)
# Action form validates name and description.
workflow.name = request.POST.get("name")
workflow.description = request.POST.get("description")
workflow.save()
doc = workflow.doc.get()
doc.extra = "jobsub"
doc.save()
# Save design again to update all fields.
data = format_dict_field_values(request.POST.copy())
_save_design(workflow.id, data)
return get_design(request, workflow.id)
def edit_action(request, action):
ActionForm = design_form_by_type(action.node_type)
if request.method == 'POST' and Job.objects.can_edit_or_exception(
request, action.workflow):
action_form = ActionForm(request.POST, instance=action)
if action_form.is_valid():
action = action_form.save()
return redirect(
reverse('oozie:edit_workflow',
kwargs={'workflow': action.workflow.id}))
else:
action_form = ActionForm(instance=action)
return render(
'editor/edit_workflow_action.mako', request, {
'workflow':
action.workflow,
'job_properties':
'job_properties' in action_form.fields
and extract_field_data(action_form['job_properties']) or '[]',
'files':
'files' in action_form.fields
and extract_field_data(action_form['files']) or '[]',
'archives':
'archives' in action_form.fields
and extract_field_data(action_form['archives']) or '[]',
'params':
'params' in action_form.fields
and extract_field_data(action_form['params']) or '[]',
'prepares':
'prepares' in action_form.fields
and extract_field_data(action_form['prepares']) or '[]',
'action_form':
action_form,
'node_type':
action.node_type,
'properties_hint':
_STD_PROPERTIES_JSON,
'form_url':
reverse('oozie:edit_action', kwargs={'action': action.id}),
'can_edit_action':
action.workflow.is_editable(request.user)
})
def edit_workflow(request, workflow):
history = History.objects.filter(submitter=request.user,
job=workflow).order_by('-submission_date')
workflow_form = WorkflowForm(instance=workflow)
user_can_access_job = workflow.can_read(request.user)
user_can_edit_job = workflow.is_editable(request.user)
oozie_api = get_oozie(request.user)
credentials = Credentials()
credentials.fetch(oozie_api)
return render(
'editor/edit_workflow.mako', request, {
'oozie_api':
oozie_api,
'workflow_form':
workflow_form,
'workflow':
workflow,
'history':
history,
'user_can_access_job':
user_can_access_job,
'user_can_edit_job':
user_can_edit_job,
'job_properties':
extract_field_data(workflow_form['job_properties']),
'link_form':
LinkForm(),
'default_link_form':
DefaultLinkForm(action=workflow.start),
'node_form':
NodeForm(),
'action_forms':
[(node_type, design_form_by_type(node_type, request.user,
workflow)())
for node_type in ACTION_TYPES.keys()],
'credentials':
json.dumps(list(credentials.credentials.keys()))
})
def edit_workflow(request, workflow):
history = History.objects.filter(submitter=request.user, job=workflow).order_by('-submission_date')
workflow_form = WorkflowForm(instance=workflow)
user_can_access_job = workflow.can_read(request.user)
user_can_edit_job = workflow.is_editable(request.user)
api = get_oozie(request.user)
credentials = Credentials()
credentials.fetch(api)
return render('editor/edit_workflow.mako', request, {
'workflow_form': workflow_form,
'workflow': workflow,
'history': history,
'user_can_access_job': user_can_access_job,
'user_can_edit_job': user_can_edit_job,
'job_properties': extract_field_data(workflow_form['job_properties']),
'link_form': LinkForm(),
'default_link_form': DefaultLinkForm(action=workflow.start),
'node_form': NodeForm(),
'action_forms': [(node_type, design_form_by_type(node_type, request.user, workflow)())
for node_type in ACTION_TYPES.iterkeys()],
'credentials': json.dumps(credentials.credentials.keys())
})
def edit_action(request, action):
ActionForm = design_form_by_type(action.node_type)
if request.method == 'POST' and Job.objects.can_edit_or_exception(request, action.workflow):
action_form = ActionForm(request.POST, instance=action)
if action_form.is_valid():
action = action_form.save()
return redirect(reverse('oozie:edit_workflow', kwargs={'workflow': action.workflow.id}))
else:
action_form = ActionForm(instance=action)
return render('editor/edit_workflow_action.mako', request, {
'workflow': action.workflow,
'job_properties': 'job_properties' in action_form.fields and extract_field_data(action_form['job_properties']) or '[]',
'files': 'files' in action_form.fields and extract_field_data(action_form['files']) or '[]',
'archives': 'archives' in action_form.fields and extract_field_data(action_form['archives']) or '[]',
'params': 'params' in action_form.fields and extract_field_data(action_form['params']) or '[]',
'prepares': 'prepares' in action_form.fields and extract_field_data(action_form['prepares']) or '[]',
'action_form': action_form,
'node_type': action.node_type,
'properties_hint': _STD_PROPERTIES_JSON,
'form_url': reverse('oozie:edit_action', kwargs={'action': action.id}),
'can_edit_action': action.workflow.is_editable(request.user)
})
