def _validate_nodes_json(json_nodes, errors, user, workflow): """ Validates every node and link in the workflow. node_type is the node type of the action information passed. node_dict is a dictionary describing the node. errors is a dictionary that will be populated with any found errors. user is a User object that is associated with the node_type. Only needed for Subworkflow node. workflow is the Workflow object associated with the node. Only needed for Subworkflow node. Returns Boolean. """ assert isinstance(errors, dict), "errors must be a dict." result = True for node in json_nodes: _errors = {} node_dict = format_dict_field_values(node) if node['node_type'] in ACTION_TYPES: node_result = _validate_node_json(node['node_type'], node_dict, _errors, user, workflow) else: node_result = True link_result = _validate_node_links_json(node['node_type'], node_dict['child_links'], _errors) result = result and node_result and link_result if not node.has_key('name') and ( not node.has_key('node_type') or not node.has_key('id') ): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': 'Node is missing a name.'}, error_code=400) errors[node.get('name', '%s-%s' % ( node.get('node_type'), node.get('id')))] = _errors return result
def _validate_nodes_json(json_nodes, errors, user, workflow): """ Validates every node and link in the workflow. node_type is the node type of the action information passed. node_dict is a dictionary describing the node. errors is a dictionary that will be populated with any found errors. user is a User object that is associated with the node_type. Only needed for Subworkflow node. workflow is the Workflow object associated with the node. Only needed for Subworkflow node. Returns Boolean. """ assert isinstance(errors, dict), "errors must be a dict." result = True for node in json_nodes: _errors = {} node_dict = format_dict_field_values(node) if node['node_type'] in ACTION_TYPES: node_result = _validate_node_json(node['node_type'], node_dict, _errors, user, workflow) else: node_result = True link_result = _validate_node_links_json(node['node_type'], node_dict['child_links'], _errors) result = result and node_result and link_result if 'name' not in node and ( 'node_type' not in node or 'id' not in node ): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': 'Node is missing a name.'}, error_code=400) errors[node.get('name', '%s-%s' % ( node.get('node_type'), node.get('id')))] = _errors return result
def new_design(request, node_type): """ Designs are the interpolation of Workflows and a single action. Save ``name`` and ``description`` of workflows. Also, use ``id`` of workflows. """ if request.method != 'POST': raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405) workflow = Workflow.objects.new_workflow(request.user) ActionForm = design_form_by_type(node_type, request.user, workflow) form = ActionForm(request.POST) if not form.is_valid(): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400) workflow.managed = False # Every one should be able to execute and clone a design. workflow.is_shared = True workflow.save() Workflow.objects.initialize(workflow, request.fs) action = form.save(commit=False) action.workflow = workflow action.node_type = node_type action.save() workflow.start.add_node(action) action.add_node(workflow.end) workflow.name = request.POST.get('name') workflow.description = request.POST.get('description') workflow.save() data = format_dict_field_values(request.POST.copy()) data['id'] = workflow.id return render_json(data)
def save_design(request, design_id): workflow = _get_design(design_id) _check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id}) ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow) form = ActionForm(request.POST) if not form.is_valid(): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400) data = format_dict_field_values(request.POST.copy()) sanitize_node_dict(data) workflow.name = data['name'] workflow.description = data['description'] node = workflow.start.get_child('to').get_full_node() node_id = node.id for key in data: setattr(node, key, data[key]) node.id = node_id node.pk = node_id node.save() workflow.save() data['id'] = workflow.id return render_json(data);
def workflow_save(request, workflow): if request.method != 'POST': raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405) json_workflow = format_dict_field_values( json.loads(request.POST.get('workflow'))) json_workflow.setdefault('schema_version', workflow.schema_version) form = WorkflowForm(data=json_workflow) if not form.is_valid(): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': form.errors}, error_code=400) json_nodes = json_workflow['nodes'] id_map = {} errors = {} if not _validate_nodes_json(json_nodes, errors, request.user, workflow): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': errors}, error_code=400) workflow = _update_workflow_json(json_workflow) nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user) # Update links index = 0 for json_node in json_nodes: child_links = json_node['child_links'] Link.objects.filter(parent=nodes[index]).delete() for child_link in child_links: link = Link() link.id = getattr(child_link, 'id', None) link.name = child_link['name'] id = str(child_link['parent']) link.parent = Node.objects.get(id=id_map[id]) id = str(child_link['child']) link.child = Node.objects.get(id=id_map[id]) link.comment = child_link.get('comment', '') link.save() index += 1 # Make sure workflow HDFS permissions are correct Workflow.objects.check_workspace(workflow, request.fs) return _workflow(request, workflow=workflow)
def run_jar(jar_hdfs_path, java_args, main_class, wf_name=None, description=None, username='******'): user, created = User.objects.get_or_create(username=username) url = WEBHDFS_URL fs_defaultfs = FS_DEFAULTFS fs = webhdfs.WebHdfs(url, fs_defaultfs) if not wf_name: wf_name = jar_hdfs_path if not description: description = java_args POST = {u'files': u'[]', u'is_shared': u'true', u'args': java_args, u'name': wf_name, u'parameters': u'[{"name":"oozie.use.system.libpath","value":"true"}]', u'workflow': u'0', u'java_opts': u'', u'jar_path': jar_hdfs_path, u'job_properties': u'[]', u'job_xml': u'', u'main_class': main_class, u'prepares': u'[]', u'node_type': u'java', u'last_modified': u'0', u'archives': u'[]', u'owner': u'', u'capture_output': u'false', u'id': u'', u'description': description} node_type = 'java' workflow = Workflow.objects.new_workflow(user) ActionForm = design_form_by_type(node_type, user, workflow) form = ActionForm(POST) workflow.managed = False # Every one should be able to execute and clone a design. workflow.is_shared = True workflow.save() Workflow.objects.initialize(workflow, fs) action = form.save(commit=False) action.workflow = workflow action.node_type = node_type action.save() workflow.start.add_node(action) action.add_node(workflow.end) # Action form validates name and description. workflow.name = POST.get('name') workflow.description = POST.get('description') workflow.save() #save_design data = format_dict_field_values(POST.copy()) save_design_my(workflow.id, data) # after saving workflow we run it submission = Submission(user, workflow, fs) mapping = {u'oozie.use.system.libpath': u'true'} #__import__("ipdb").set_trace() job_id = None attempts = ATTEMPTS_NUM while attempts > 0: try: job_id = submit_workflow_my(user, fs, workflow, mapping) break except PopupException, ex: __import__("ipdb").set_trace() print str(attempts) + ", " + str(ex) attempts -= 1
def workflow_save(request, workflow): if request.method != "POST": raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_("Must be POST request."), error_code=405) json_workflow = format_dict_field_values(json.loads(request.POST.get("workflow"))) json_workflow.setdefault("schema_version", workflow.schema_version) form = WorkflowForm(data=json_workflow) if not form.is_valid(): raise StructuredException( code="INVALID_REQUEST_ERROR", message=_("Error saving workflow"), data={"errors": form.errors}, error_code=400, ) json_nodes = json_workflow["nodes"] id_map = {} errors = {} if not _validate_nodes_json(json_nodes, errors, request.user, workflow): raise StructuredException( code="INVALID_REQUEST_ERROR", message=_("Error saving workflow"), data={"errors": errors}, error_code=400 ) workflow = _update_workflow_json(json_workflow) nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user) # Update links index = 0 for json_node in json_nodes: child_links = json_node["child_links"] Link.objects.filter(parent=nodes[index]).delete() for child_link in child_links: link = Link() link.id = getattr(child_link, "id", None) link.name = child_link["name"] id = str(child_link["parent"]) link.parent = Node.objects.get(id=id_map[id]) id = str(child_link["child"]) link.child = Node.objects.get(id=id_map[id]) link.comment = child_link.get("comment", "") link.save() index += 1 # Make sure workflow HDFS permissions are correct Workflow.objects.check_workspace(workflow, request.fs) return _workflow(request, workflow=workflow)
def workflow_validate_node(request, workflow, node_type): response = {'status': -1, 'data': {}} node_dict = format_dict_field_values(json.loads(request.POST.get('node'))) if _validate_node_json(node_type, node_dict, response['data'], request.user, workflow): response['status'] = 0 else: response['status'] = -1 return HttpResponse(json.dumps(response), mimetype="application/json")
def workflow_validate_node(request, workflow, node_type): response = {'status': -1, 'data': {}} node_dict = format_dict_field_values(json.loads(request.POST.get('node'))) if _validate_node_json(node_type, node_dict, response['data'], request.user, workflow): response['status'] = 0 else: response['status'] = -1 return JsonResponse(response)
def workflow_validate_node(request, workflow, node_type): response = {'status': -1, 'data': {}} node_dict = format_dict_field_values(json.loads(str(request.POST.get('node')))) if _validate_node_json(node_type, node_dict, response['data'], request.user, workflow): response['status'] = 0 else: response['status'] = -1 return HttpResponse(json.dumps(response), mimetype="application/json")
def workflow_validate_node(request, workflow, node_type): response = {"status": -1, "data": {}} node_dict = format_dict_field_values(json.loads(request.POST.get("node"))) if _validate_node_json(node_type, node_dict, response["data"], request.user, workflow): response["status"] = 0 else: response["status"] = -1 return JsonResponse(response)
def save_design(request, design_id): workflow = _get_design(design_id) _check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id}) ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow) form = ActionForm(request.POST) if not form.is_valid(): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400) data = format_dict_field_values(request.POST.copy()) _save_design(design_id, data) return get_design(request, design_id);
def workflow_save(request, workflow): if request.method != 'POST': raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405) json_workflow = format_dict_field_values(json.loads(str(request.POST.get('workflow')))) json_workflow.setdefault('schema_version', workflow.schema_version) form = WorkflowForm(data=json_workflow) if not form.is_valid(): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': form.errors}, error_code=400) json_nodes = json_workflow['nodes'] id_map = {} errors = {} if not _validate_nodes_json(json_nodes, errors, request.user, workflow): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': errors}, error_code=400) workflow = _update_workflow_json(json_workflow) nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user) # Update links index = 0 for json_node in json_nodes: child_links = json_node['child_links'] Link.objects.filter(parent=nodes[index]).delete() for child_link in child_links: link = Link() link.id = getattr(child_link, 'id', None) link.name = child_link['name'] id = str(child_link['parent']) link.parent = Node.objects.get(id=id_map[id]) id = str(child_link['child']) link.child = Node.objects.get(id=id_map[id]) link.comment = child_link.get('comment', '') link.save() index += 1 # Make sure workflow HDFS permissions are correct Workflow.objects.check_workspace(workflow, request.fs) return _workflow(request, workflow=workflow)
def new_design(request, node_type): """ Designs are the interpolation of Workflows and a single action. Save ``name`` and ``description`` of workflows. Also, use ``id`` of workflows. """ if request.method != 'POST': raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405) workflow = Workflow.objects.new_workflow(request.user) ActionForm = design_form_by_type(node_type, request.user, workflow) form = ActionForm(request.POST) if not form.is_valid(): raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400) workflow.managed = False # Every one should be able to execute and clone a design. workflow.is_shared = True workflow.save() Workflow.objects.initialize(workflow, request.fs) action = form.save(commit=False) action.workflow = workflow action.node_type = node_type action.save() workflow.start.add_node(action) action.add_node(workflow.end) # Action form validates name and description. workflow.name = request.POST.get('name') workflow.description = request.POST.get('description') workflow.save() doc = workflow.doc.get() doc.extra = 'jobsub' doc.save() # Save design again to update all fields. data = format_dict_field_values(request.POST.copy()) _save_design(workflow.id, data) return get_design(request, workflow.id)
def new_design(request, node_type): """ Designs are the interpolation of Workflows and a single action. Save ``name`` and ``description`` of workflows. Also, use ``id`` of workflows. """ if request.method != "POST": raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_("Must be POST request."), error_code=405) workflow = Workflow.objects.new_workflow(request.user) ActionForm = design_form_by_type(node_type, request.user, workflow) form = ActionForm(request.POST) if not form.is_valid(): raise StructuredException( code="INVALID_REQUEST_ERROR", message=_("Error saving design"), data={"errors": form.errors}, error_code=400 ) workflow.managed = False # Every one should be able to execute and clone a design. workflow.is_shared = True workflow.save() Workflow.objects.initialize(workflow, request.fs) action = form.save(commit=False) action.workflow = workflow action.node_type = node_type action.save() workflow.start.add_node(action) action.add_node(workflow.end) # Action form validates name and description. workflow.name = request.POST.get("name") workflow.description = request.POST.get("description") workflow.save() doc = workflow.doc.get() doc.extra = "jobsub" doc.save() # Save design again to update all fields. data = format_dict_field_values(request.POST.copy()) _save_design(workflow.id, data) return get_design(request, workflow.id)