def _validate_nodes_json(json_nodes, errors, user, workflow):
"""
Validates every node and link in the workflow.
node_type is the node type of the action information passed.
node_dict is a dictionary describing the node.
errors is a dictionary that will be populated with any found errors.
user is a User object that is associated with the node_type. Only needed for Subworkflow node.
workflow is the Workflow object associated with the node. Only needed for Subworkflow node.
Returns Boolean.
"""
assert isinstance(errors, dict), "errors must be a dict."
result = True
for node in json_nodes:
_errors = {}
node_dict = format_dict_field_values(node)
if node['node_type'] in ACTION_TYPES:
node_result = _validate_node_json(node['node_type'], node_dict, _errors, user, workflow)
else:
node_result = True
link_result = _validate_node_links_json(node['node_type'], node_dict['child_links'], _errors)
result = result and node_result and link_result
if not node.has_key('name') and ( not node.has_key('node_type') or not node.has_key('id') ):
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': 'Node is missing a name.'}, error_code=400)
errors[node.get('name', '%s-%s' % ( node.get('node_type'), node.get('id')))] = _errors
return result
def _validate_nodes_json(json_nodes, errors, user, workflow):
"""
Validates every node and link in the workflow.
node_type is the node type of the action information passed.
node_dict is a dictionary describing the node.
errors is a dictionary that will be populated with any found errors.
user is a User object that is associated with the node_type. Only needed for Subworkflow node.
workflow is the Workflow object associated with the node. Only needed for Subworkflow node.
Returns Boolean.
"""
assert isinstance(errors, dict), "errors must be a dict."
result = True
for node in json_nodes:
_errors = {}
node_dict = format_dict_field_values(node)
if node['node_type'] in ACTION_TYPES:
node_result = _validate_node_json(node['node_type'], node_dict, _errors, user, workflow)
else:
node_result = True
link_result = _validate_node_links_json(node['node_type'], node_dict['child_links'], _errors)
result = result and node_result and link_result
if 'name' not in node and ( 'node_type' not in node or 'id' not in node ):
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': 'Node is missing a name.'}, error_code=400)
errors[node.get('name', '%s-%s' % ( node.get('node_type'), node.get('id')))] = _errors
return result
def new_design(request, node_type):
"""
Designs are the interpolation of Workflows and a single action.
Save ``name`` and ``description`` of workflows.
Also, use ``id`` of workflows.
"""
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405)
workflow = Workflow.objects.new_workflow(request.user)
ActionForm = design_form_by_type(node_type, request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)
workflow.managed = False
# Every one should be able to execute and clone a design.
workflow.is_shared = True
workflow.save()
Workflow.objects.initialize(workflow, request.fs)
action = form.save(commit=False)
action.workflow = workflow
action.node_type = node_type
action.save()
workflow.start.add_node(action)
action.add_node(workflow.end)
workflow.name = request.POST.get('name')
workflow.description = request.POST.get('description')
workflow.save()
data = format_dict_field_values(request.POST.copy())
data['id'] = workflow.id
return render_json(data)
def save_design(request, design_id):
workflow = _get_design(design_id)
_check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id})
ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)
data = format_dict_field_values(request.POST.copy())
sanitize_node_dict(data)
workflow.name = data['name']
workflow.description = data['description']
node = workflow.start.get_child('to').get_full_node()
node_id = node.id
for key in data:
setattr(node, key, data[key])
node.id = node_id
node.pk = node_id
node.save()
workflow.save()
data['id'] = workflow.id
return render_json(data);
def workflow_save(request, workflow):
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be POST request.'),
error_code=405)
json_workflow = format_dict_field_values(
json.loads(request.POST.get('workflow')))
json_workflow.setdefault('schema_version', workflow.schema_version)
form = WorkflowForm(data=json_workflow)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving workflow'),
data={'errors': form.errors},
error_code=400)
json_nodes = json_workflow['nodes']
id_map = {}
errors = {}
if not _validate_nodes_json(json_nodes, errors, request.user, workflow):
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving workflow'),
data={'errors': errors},
error_code=400)
workflow = _update_workflow_json(json_workflow)
nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map,
request.user)
# Update links
index = 0
for json_node in json_nodes:
child_links = json_node['child_links']
Link.objects.filter(parent=nodes[index]).delete()
for child_link in child_links:
link = Link()
link.id = getattr(child_link, 'id', None)
link.name = child_link['name']
id = str(child_link['parent'])
link.parent = Node.objects.get(id=id_map[id])
id = str(child_link['child'])
link.child = Node.objects.get(id=id_map[id])
link.comment = child_link.get('comment', '')
link.save()
index += 1
# Make sure workflow HDFS permissions are correct
Workflow.objects.check_workspace(workflow, request.fs)
return _workflow(request, workflow=workflow)
def run_jar(jar_hdfs_path, java_args, main_class, wf_name=None, description=None, username='******'):
user, created = User.objects.get_or_create(username=username)
url = WEBHDFS_URL
fs_defaultfs = FS_DEFAULTFS
fs = webhdfs.WebHdfs(url, fs_defaultfs)
if not wf_name:
wf_name = jar_hdfs_path
if not description:
description = java_args
POST = {u'files': u'[]', u'is_shared': u'true',
u'args': java_args,
u'name': wf_name, u'parameters': u'[{"name":"oozie.use.system.libpath","value":"true"}]',
u'workflow': u'0', u'java_opts': u'', u'jar_path': jar_hdfs_path, u'job_properties': u'[]',
u'job_xml': u'', u'main_class': main_class, u'prepares': u'[]',
u'node_type': u'java', u'last_modified': u'0', u'archives': u'[]', u'owner': u'',
u'capture_output': u'false', u'id': u'', u'description': description}
node_type = 'java'
workflow = Workflow.objects.new_workflow(user)
ActionForm = design_form_by_type(node_type, user, workflow)
form = ActionForm(POST)
workflow.managed = False
# Every one should be able to execute and clone a design.
workflow.is_shared = True
workflow.save()
Workflow.objects.initialize(workflow, fs)
action = form.save(commit=False)
action.workflow = workflow
action.node_type = node_type
action.save()
workflow.start.add_node(action)
action.add_node(workflow.end)
# Action form validates name and description.
workflow.name = POST.get('name')
workflow.description = POST.get('description')
workflow.save()
#save_design
data = format_dict_field_values(POST.copy())
save_design_my(workflow.id, data)
# after saving workflow we run it
submission = Submission(user, workflow, fs)
mapping = {u'oozie.use.system.libpath': u'true'}
#__import__("ipdb").set_trace()
job_id = None
attempts = ATTEMPTS_NUM
while attempts > 0:
try:
job_id = submit_workflow_my(user, fs, workflow, mapping)
break
except PopupException, ex:
__import__("ipdb").set_trace()
print str(attempts) + ", " + str(ex)
attempts -= 1
def workflow_save(request, workflow):
if request.method != "POST":
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_("Must be POST request."), error_code=405)
json_workflow = format_dict_field_values(json.loads(request.POST.get("workflow")))
json_workflow.setdefault("schema_version", workflow.schema_version)
form = WorkflowForm(data=json_workflow)
if not form.is_valid():
raise StructuredException(
code="INVALID_REQUEST_ERROR",
message=_("Error saving workflow"),
data={"errors": form.errors},
error_code=400,
)
json_nodes = json_workflow["nodes"]
id_map = {}
errors = {}
if not _validate_nodes_json(json_nodes, errors, request.user, workflow):
raise StructuredException(
code="INVALID_REQUEST_ERROR", message=_("Error saving workflow"), data={"errors": errors}, error_code=400
)
workflow = _update_workflow_json(json_workflow)
nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user)
# Update links
index = 0
for json_node in json_nodes:
child_links = json_node["child_links"]
Link.objects.filter(parent=nodes[index]).delete()
for child_link in child_links:
link = Link()
link.id = getattr(child_link, "id", None)
link.name = child_link["name"]
id = str(child_link["parent"])
link.parent = Node.objects.get(id=id_map[id])
id = str(child_link["child"])
link.child = Node.objects.get(id=id_map[id])
link.comment = child_link.get("comment", "")
link.save()
index += 1
# Make sure workflow HDFS permissions are correct
Workflow.objects.check_workspace(workflow, request.fs)
return _workflow(request, workflow=workflow)
def workflow_validate_node(request, workflow, node_type):
response = {'status': -1, 'data': {}}
node_dict = format_dict_field_values(json.loads(request.POST.get('node')))
if _validate_node_json(node_type, node_dict, response['data'], request.user, workflow):
response['status'] = 0
else:
response['status'] = -1
return HttpResponse(json.dumps(response), mimetype="application/json")
def workflow_validate_node(request, workflow, node_type):
response = {'status': -1, 'data': {}}
node_dict = format_dict_field_values(json.loads(request.POST.get('node')))
if _validate_node_json(node_type, node_dict, response['data'], request.user, workflow):
response['status'] = 0
else:
response['status'] = -1
return JsonResponse(response)
def workflow_validate_node(request, workflow, node_type):
response = {'status': -1, 'data': {}}
node_dict = format_dict_field_values(json.loads(str(request.POST.get('node'))))
if _validate_node_json(node_type, node_dict, response['data'], request.user, workflow):
response['status'] = 0
else:
response['status'] = -1
return HttpResponse(json.dumps(response), mimetype="application/json")
def workflow_validate_node(request, workflow, node_type):
response = {"status": -1, "data": {}}
node_dict = format_dict_field_values(json.loads(request.POST.get("node")))
if _validate_node_json(node_type, node_dict, response["data"], request.user, workflow):
response["status"] = 0
else:
response["status"] = -1
return JsonResponse(response)
def save_design(request, design_id):
workflow = _get_design(design_id)
_check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id})
ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)
data = format_dict_field_values(request.POST.copy())
_save_design(design_id, data)
return get_design(request, design_id);
def save_design(request, design_id):
workflow = _get_design(design_id)
_check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id})
ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)
data = format_dict_field_values(request.POST.copy())
_save_design(design_id, data)
return get_design(request, design_id);
def workflow_save(request, workflow):
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405)
json_workflow = format_dict_field_values(json.loads(str(request.POST.get('workflow'))))
json_workflow.setdefault('schema_version', workflow.schema_version)
form = WorkflowForm(data=json_workflow)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': form.errors}, error_code=400)
json_nodes = json_workflow['nodes']
id_map = {}
errors = {}
if not _validate_nodes_json(json_nodes, errors, request.user, workflow):
raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': errors}, error_code=400)
workflow = _update_workflow_json(json_workflow)
nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user)
# Update links
index = 0
for json_node in json_nodes:
child_links = json_node['child_links']
Link.objects.filter(parent=nodes[index]).delete()
for child_link in child_links:
link = Link()
link.id = getattr(child_link, 'id', None)
link.name = child_link['name']
id = str(child_link['parent'])
link.parent = Node.objects.get(id=id_map[id])
id = str(child_link['child'])
link.child = Node.objects.get(id=id_map[id])
link.comment = child_link.get('comment', '')
link.save()
index += 1
# Make sure workflow HDFS permissions are correct
Workflow.objects.check_workspace(workflow, request.fs)
return _workflow(request, workflow=workflow)
def new_design(request, node_type):
"""
Designs are the interpolation of Workflows and a single action.
Save ``name`` and ``description`` of workflows.
Also, use ``id`` of workflows.
"""
if request.method != 'POST':
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
message=_('Must be POST request.'),
error_code=405)
workflow = Workflow.objects.new_workflow(request.user)
ActionForm = design_form_by_type(node_type, request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(code="INVALID_REQUEST_ERROR",
message=_('Error saving design'),
data={'errors': form.errors},
error_code=400)
workflow.managed = False
# Every one should be able to execute and clone a design.
workflow.is_shared = True
workflow.save()
Workflow.objects.initialize(workflow, request.fs)
action = form.save(commit=False)
action.workflow = workflow
action.node_type = node_type
action.save()
workflow.start.add_node(action)
action.add_node(workflow.end)
# Action form validates name and description.
workflow.name = request.POST.get('name')
workflow.description = request.POST.get('description')
workflow.save()
doc = workflow.doc.get()
doc.extra = 'jobsub'
doc.save()
# Save design again to update all fields.
data = format_dict_field_values(request.POST.copy())
_save_design(workflow.id, data)
return get_design(request, workflow.id)
def new_design(request, node_type):
"""
Designs are the interpolation of Workflows and a single action.
Save ``name`` and ``description`` of workflows.
Also, use ``id`` of workflows.
"""
if request.method != "POST":
raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_("Must be POST request."), error_code=405)
workflow = Workflow.objects.new_workflow(request.user)
ActionForm = design_form_by_type(node_type, request.user, workflow)
form = ActionForm(request.POST)
if not form.is_valid():
raise StructuredException(
code="INVALID_REQUEST_ERROR", message=_("Error saving design"), data={"errors": form.errors}, error_code=400
)
workflow.managed = False
# Every one should be able to execute and clone a design.
workflow.is_shared = True
workflow.save()
Workflow.objects.initialize(workflow, request.fs)
action = form.save(commit=False)
action.workflow = workflow
action.node_type = node_type
action.save()
workflow.start.add_node(action)
action.add_node(workflow.end)
# Action form validates name and description.
workflow.name = request.POST.get("name")
workflow.description = request.POST.get("description")
workflow.save()
doc = workflow.doc.get()
doc.extra = "jobsub"
doc.save()
# Save design again to update all fields.
data = format_dict_field_values(request.POST.copy())
_save_design(workflow.id, data)
return get_design(request, workflow.id)
