Beispiel #1
0
def _validate_nodes_json(json_nodes, errors, user, workflow):
  """
  Validates every node and link in the workflow.
  node_type is the node type of the action information passed.
  node_dict is a dictionary describing the node.
  errors is a dictionary that will be populated with any found errors.
  user is a User object that is associated with the node_type. Only needed for Subworkflow node.
  workflow is the Workflow object associated with the node. Only needed for Subworkflow node.
  Returns Boolean.
  """
  assert isinstance(errors, dict), "errors must be a dict."
  result = True

  for node in json_nodes:
    _errors = {}
    node_dict = format_dict_field_values(node)
    if node['node_type'] in ACTION_TYPES:
      node_result = _validate_node_json(node['node_type'], node_dict, _errors, user, workflow)
    else:
      node_result = True
    link_result = _validate_node_links_json(node['node_type'], node_dict['child_links'], _errors)
    result = result and node_result and link_result
    if not node.has_key('name') and ( not node.has_key('node_type') or not node.has_key('id') ):
      raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': 'Node is missing a name.'}, error_code=400)
    errors[node.get('name', '%s-%s' % ( node.get('node_type'), node.get('id')))] = _errors

  return result
Beispiel #2
0
def _validate_nodes_json(json_nodes, errors, user, workflow):
  """
  Validates every node and link in the workflow.
  node_type is the node type of the action information passed.
  node_dict is a dictionary describing the node.
  errors is a dictionary that will be populated with any found errors.
  user is a User object that is associated with the node_type. Only needed for Subworkflow node.
  workflow is the Workflow object associated with the node. Only needed for Subworkflow node.
  Returns Boolean.
  """
  assert isinstance(errors, dict), "errors must be a dict."
  result = True

  for node in json_nodes:
    _errors = {}
    node_dict = format_dict_field_values(node)
    if node['node_type'] in ACTION_TYPES:
      node_result = _validate_node_json(node['node_type'], node_dict, _errors, user, workflow)
    else:
      node_result = True
    link_result = _validate_node_links_json(node['node_type'], node_dict['child_links'], _errors)
    result = result and node_result and link_result
    if 'name' not in node and ( 'node_type' not in node or 'id' not in node ):
      raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': 'Node is missing a name.'}, error_code=400)
    errors[node.get('name', '%s-%s' % ( node.get('node_type'), node.get('id')))] = _errors

  return result
Beispiel #3
0
def new_design(request, node_type):
  """
  Designs are the interpolation of Workflows and a single action.
  Save ``name`` and ``description`` of workflows.
  Also, use ``id`` of workflows.
  """
  if request.method != 'POST':
    raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405)

  workflow = Workflow.objects.new_workflow(request.user)
  ActionForm = design_form_by_type(node_type, request.user, workflow)
  form = ActionForm(request.POST)

  if not form.is_valid():
    raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)

  workflow.managed = False
  # Every one should be able to execute and clone a design.
  workflow.is_shared = True
  workflow.save()
  Workflow.objects.initialize(workflow, request.fs)
  action = form.save(commit=False)
  action.workflow = workflow
  action.node_type = node_type
  action.save()
  workflow.start.add_node(action)
  action.add_node(workflow.end)
  workflow.name = request.POST.get('name')
  workflow.description = request.POST.get('description')
  workflow.save()

  data = format_dict_field_values(request.POST.copy())
  data['id'] = workflow.id
  return render_json(data)
Beispiel #4
0
def save_design(request, design_id):
  workflow = _get_design(design_id)
  _check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id})

  ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow)
  form = ActionForm(request.POST)

  if not form.is_valid():
    raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)

  data = format_dict_field_values(request.POST.copy())
  sanitize_node_dict(data)
  workflow.name = data['name']
  workflow.description = data['description']
  node = workflow.start.get_child('to').get_full_node()
  node_id = node.id
  for key in data:
    setattr(node, key, data[key])
  node.id = node_id
  node.pk = node_id
  node.save()
  workflow.save()

  data['id'] = workflow.id
  return render_json(data);
Beispiel #5
0
def workflow_save(request, workflow):
    if request.method != 'POST':
        raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
                                  message=_('Must be POST request.'),
                                  error_code=405)

    json_workflow = format_dict_field_values(
        json.loads(request.POST.get('workflow')))
    json_workflow.setdefault('schema_version', workflow.schema_version)

    form = WorkflowForm(data=json_workflow)

    if not form.is_valid():
        raise StructuredException(code="INVALID_REQUEST_ERROR",
                                  message=_('Error saving workflow'),
                                  data={'errors': form.errors},
                                  error_code=400)

    json_nodes = json_workflow['nodes']
    id_map = {}
    errors = {}

    if not _validate_nodes_json(json_nodes, errors, request.user, workflow):
        raise StructuredException(code="INVALID_REQUEST_ERROR",
                                  message=_('Error saving workflow'),
                                  data={'errors': errors},
                                  error_code=400)

    workflow = _update_workflow_json(json_workflow)
    nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map,
                                        request.user)

    # Update links
    index = 0
    for json_node in json_nodes:
        child_links = json_node['child_links']
        Link.objects.filter(parent=nodes[index]).delete()

        for child_link in child_links:
            link = Link()
            link.id = getattr(child_link, 'id', None)
            link.name = child_link['name']

            id = str(child_link['parent'])
            link.parent = Node.objects.get(id=id_map[id])

            id = str(child_link['child'])
            link.child = Node.objects.get(id=id_map[id])

            link.comment = child_link.get('comment', '')

            link.save()

        index += 1

    # Make sure workflow HDFS permissions are correct
    Workflow.objects.check_workspace(workflow, request.fs)

    return _workflow(request, workflow=workflow)
def run_jar(jar_hdfs_path, java_args, main_class, wf_name=None, description=None, username='******'):
    user, created = User.objects.get_or_create(username=username)
    url = WEBHDFS_URL
    fs_defaultfs = FS_DEFAULTFS
    fs = webhdfs.WebHdfs(url, fs_defaultfs)
    if not wf_name:
        wf_name = jar_hdfs_path
    if not description:
        description = java_args
    POST = {u'files': u'[]', u'is_shared': u'true',
            u'args': java_args,
            u'name': wf_name, u'parameters': u'[{"name":"oozie.use.system.libpath","value":"true"}]',
            u'workflow': u'0', u'java_opts': u'', u'jar_path': jar_hdfs_path, u'job_properties': u'[]',
            u'job_xml': u'', u'main_class': main_class, u'prepares': u'[]',
            u'node_type': u'java', u'last_modified': u'0', u'archives': u'[]', u'owner': u'',
            u'capture_output': u'false', u'id': u'', u'description': description}
    node_type = 'java'
    workflow = Workflow.objects.new_workflow(user)
    ActionForm = design_form_by_type(node_type, user, workflow)
    form = ActionForm(POST)

    workflow.managed = False
    # Every one should be able to execute and clone a design.
    workflow.is_shared = True
    workflow.save()
    Workflow.objects.initialize(workflow, fs)
    action = form.save(commit=False)
    action.workflow = workflow
    action.node_type = node_type
    action.save()
    workflow.start.add_node(action)
    action.add_node(workflow.end)

    # Action form validates name and description.
    workflow.name = POST.get('name')
    workflow.description = POST.get('description')
    workflow.save()

    #save_design
    data = format_dict_field_values(POST.copy())
    save_design_my(workflow.id, data)

    # after saving workflow we run it
    submission = Submission(user, workflow, fs)
    mapping = {u'oozie.use.system.libpath': u'true'}
    #__import__("ipdb").set_trace()
    job_id = None
    attempts = ATTEMPTS_NUM
    while attempts > 0:
        try:
            job_id = submit_workflow_my(user, fs, workflow, mapping)
            break
        except PopupException, ex:
            __import__("ipdb").set_trace()
            print str(attempts) + ", " + str(ex)
            attempts -= 1
Beispiel #7
0
def workflow_save(request, workflow):
    if request.method != "POST":
        raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_("Must be POST request."), error_code=405)

    json_workflow = format_dict_field_values(json.loads(request.POST.get("workflow")))
    json_workflow.setdefault("schema_version", workflow.schema_version)

    form = WorkflowForm(data=json_workflow)

    if not form.is_valid():
        raise StructuredException(
            code="INVALID_REQUEST_ERROR",
            message=_("Error saving workflow"),
            data={"errors": form.errors},
            error_code=400,
        )

    json_nodes = json_workflow["nodes"]
    id_map = {}
    errors = {}

    if not _validate_nodes_json(json_nodes, errors, request.user, workflow):
        raise StructuredException(
            code="INVALID_REQUEST_ERROR", message=_("Error saving workflow"), data={"errors": errors}, error_code=400
        )

    workflow = _update_workflow_json(json_workflow)
    nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user)

    # Update links
    index = 0
    for json_node in json_nodes:
        child_links = json_node["child_links"]
        Link.objects.filter(parent=nodes[index]).delete()

        for child_link in child_links:
            link = Link()
            link.id = getattr(child_link, "id", None)
            link.name = child_link["name"]

            id = str(child_link["parent"])
            link.parent = Node.objects.get(id=id_map[id])

            id = str(child_link["child"])
            link.child = Node.objects.get(id=id_map[id])

            link.comment = child_link.get("comment", "")

            link.save()

        index += 1

    # Make sure workflow HDFS permissions are correct
    Workflow.objects.check_workspace(workflow, request.fs)

    return _workflow(request, workflow=workflow)
Beispiel #8
0
def workflow_validate_node(request, workflow, node_type):
  response = {'status': -1, 'data': {}}

  node_dict = format_dict_field_values(json.loads(request.POST.get('node')))

  if _validate_node_json(node_type, node_dict, response['data'], request.user, workflow):
    response['status'] = 0
  else:
    response['status'] = -1

  return HttpResponse(json.dumps(response), mimetype="application/json")
Beispiel #9
0
def workflow_validate_node(request, workflow, node_type):
  response = {'status': -1, 'data': {}}

  node_dict = format_dict_field_values(json.loads(request.POST.get('node')))

  if _validate_node_json(node_type, node_dict, response['data'], request.user, workflow):
    response['status'] = 0
  else:
    response['status'] = -1

  return JsonResponse(response)
Beispiel #10
0
def workflow_validate_node(request, workflow, node_type):
  response = {'status': -1, 'data': {}}

  node_dict = format_dict_field_values(json.loads(str(request.POST.get('node'))))

  if _validate_node_json(node_type, node_dict, response['data'], request.user, workflow):
    response['status'] = 0
  else:
    response['status'] = -1

  return HttpResponse(json.dumps(response), mimetype="application/json")
Beispiel #11
0
def workflow_validate_node(request, workflow, node_type):
    response = {"status": -1, "data": {}}

    node_dict = format_dict_field_values(json.loads(request.POST.get("node")))

    if _validate_node_json(node_type, node_dict, response["data"], request.user, workflow):
        response["status"] = 0
    else:
        response["status"] = -1

    return JsonResponse(response)
Beispiel #12
0
def save_design(request, design_id):
  workflow = _get_design(design_id)
  _check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id})

  ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow)
  form = ActionForm(request.POST)

  if not form.is_valid():
    raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)

  data = format_dict_field_values(request.POST.copy())
  _save_design(design_id, data)

  return get_design(request, design_id);
Beispiel #13
0
def save_design(request, design_id):
  workflow = _get_design(design_id)
  _check_permission(request, workflow.owner.username, _("Access denied: edit design %(id)s.") % {'id': workflow.id})

  ActionForm = design_form_by_type(request.POST.get('node_type', None), request.user, workflow)
  form = ActionForm(request.POST)

  if not form.is_valid():
    raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving design'), data={'errors': form.errors}, error_code=400)

  data = format_dict_field_values(request.POST.copy())
  _save_design(design_id, data)

  return get_design(request, design_id);
Beispiel #14
0
def workflow_save(request, workflow):
  if request.method != 'POST':
    raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_('Must be POST request.'), error_code=405)

  json_workflow = format_dict_field_values(json.loads(str(request.POST.get('workflow'))))
  json_workflow.setdefault('schema_version', workflow.schema_version)

  form = WorkflowForm(data=json_workflow)

  if not form.is_valid():
    raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': form.errors}, error_code=400)

  json_nodes = json_workflow['nodes']
  id_map = {}
  errors = {}

  if not _validate_nodes_json(json_nodes, errors, request.user, workflow):
    raise StructuredException(code="INVALID_REQUEST_ERROR", message=_('Error saving workflow'), data={'errors': errors}, error_code=400)

  workflow = _update_workflow_json(json_workflow)
  nodes = _update_workflow_nodes_json(workflow, json_nodes, id_map, request.user)

  # Update links
  index = 0
  for json_node in json_nodes:
    child_links = json_node['child_links']
    Link.objects.filter(parent=nodes[index]).delete()

    for child_link in child_links:
      link = Link()
      link.id = getattr(child_link, 'id', None)
      link.name = child_link['name']

      id = str(child_link['parent'])
      link.parent = Node.objects.get(id=id_map[id])

      id = str(child_link['child'])
      link.child = Node.objects.get(id=id_map[id])

      link.comment = child_link.get('comment', '')

      link.save()

    index += 1

  # Make sure workflow HDFS permissions are correct
  Workflow.objects.check_workspace(workflow, request.fs)

  return _workflow(request, workflow=workflow)
Beispiel #15
0
def new_design(request, node_type):
    """
  Designs are the interpolation of Workflows and a single action.
  Save ``name`` and ``description`` of workflows.
  Also, use ``id`` of workflows.
  """
    if request.method != 'POST':
        raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR",
                                  message=_('Must be POST request.'),
                                  error_code=405)

    workflow = Workflow.objects.new_workflow(request.user)
    ActionForm = design_form_by_type(node_type, request.user, workflow)
    form = ActionForm(request.POST)

    if not form.is_valid():
        raise StructuredException(code="INVALID_REQUEST_ERROR",
                                  message=_('Error saving design'),
                                  data={'errors': form.errors},
                                  error_code=400)

    workflow.managed = False
    # Every one should be able to execute and clone a design.
    workflow.is_shared = True
    workflow.save()
    Workflow.objects.initialize(workflow, request.fs)
    action = form.save(commit=False)
    action.workflow = workflow
    action.node_type = node_type
    action.save()
    workflow.start.add_node(action)
    action.add_node(workflow.end)

    # Action form validates name and description.
    workflow.name = request.POST.get('name')
    workflow.description = request.POST.get('description')
    workflow.save()

    doc = workflow.doc.get()
    doc.extra = 'jobsub'
    doc.save()

    # Save design again to update all fields.
    data = format_dict_field_values(request.POST.copy())
    _save_design(workflow.id, data)

    return get_design(request, workflow.id)
Beispiel #16
0
def new_design(request, node_type):
    """
  Designs are the interpolation of Workflows and a single action.
  Save ``name`` and ``description`` of workflows.
  Also, use ``id`` of workflows.
  """
    if request.method != "POST":
        raise StructuredException(code="METHOD_NOT_ALLOWED_ERROR", message=_("Must be POST request."), error_code=405)

    workflow = Workflow.objects.new_workflow(request.user)
    ActionForm = design_form_by_type(node_type, request.user, workflow)
    form = ActionForm(request.POST)

    if not form.is_valid():
        raise StructuredException(
            code="INVALID_REQUEST_ERROR", message=_("Error saving design"), data={"errors": form.errors}, error_code=400
        )

    workflow.managed = False
    # Every one should be able to execute and clone a design.
    workflow.is_shared = True
    workflow.save()
    Workflow.objects.initialize(workflow, request.fs)
    action = form.save(commit=False)
    action.workflow = workflow
    action.node_type = node_type
    action.save()
    workflow.start.add_node(action)
    action.add_node(workflow.end)

    # Action form validates name and description.
    workflow.name = request.POST.get("name")
    workflow.description = request.POST.get("description")
    workflow.save()

    doc = workflow.doc.get()
    doc.extra = "jobsub"
    doc.save()

    # Save design again to update all fields.
    data = format_dict_field_values(request.POST.copy())
    _save_design(workflow.id, data)

    return get_design(request, workflow.id)