Beispiel #1
0
    def submit(self, pig_script, params):
        mapping = {
            'oozie.use.system.libpath': 'true',
        }

        workflow = Workflow.objects.new_workflow(self.user)

        try:
            workflow.name = OozieApi.WORKFLOW_NAME
            workflow.is_history = True
            workflow.save()
            Workflow.objects.initialize(workflow, self.fs)

            script_path = workflow.deployment_dir + '/script.pig'
            self.fs.create(script_path, data=pig_script.dict['script'])

            pig_params = []
            for param in json.loads(params):
                pig_params.append({"type": "argument", "value": "-param"})
                pig_params.append({
                    "type": "argument",
                    "value": "%(name)s=%(value)s" % param
                })

            files = []
            archives = []

            for resource in pig_script.dict['resources']:
                if resource['type'] == 'file':
                    files.append(resource['value'])
                if resource['type'] == 'archive':
                    archives.append({"dummy": "", "name": resource['value']})

            action = Pig.objects.create(
                name='pig',
                script_path=script_path,
                workflow=workflow,
                node_type='pig',
                params=json.dumps(pig_params),
                files=json.dumps(files),
                archives=json.dumps(archives),
            )

            action.add_node(workflow.end)

            start_link = workflow.start.get_link()
            start_link.child = action
            start_link.save()

            oozie_wf = _submit_workflow(self.user, self.fs, workflow, mapping)
        finally:
            workflow.delete()

        return oozie_wf
Beispiel #2
0
Datei: api.py Projekt: zlcken/hue
  def submit(self, pig_script, params):
    workflow = None

    try:
      workflow = self._create_workflow(pig_script, params)
      mapping = dict([(param['name'], param['value']) for param in workflow.get_parameters()])
      oozie_wf = _submit_workflow(self.user, self.fs, self.jt, workflow, mapping)
    finally:
      if workflow:
        workflow.delete(skip_trash=True)

    return oozie_wf
Beispiel #3
0
Datei: api.py Projekt: yjkim/hue
  def submit(self, pig_script, params):
    mapping = {
      'oozie.use.system.libpath':  'true',
    }

    workflow = Workflow.objects.new_workflow(self.user)

    try:
      workflow.name = OozieApi.WORKFLOW_NAME
      workflow.is_history = True
      workflow.save()
      Workflow.objects.initialize(workflow, self.fs)

      script_path = workflow.deployment_dir + '/script.pig'
      self.fs.create(script_path, data=pig_script.dict['script'])

      pig_params = []
      for param in json.loads(params):
        pig_params.append({"type":"argument","value":"-param"})
        pig_params.append({"type":"argument","value":"%(name)s=%(value)s" % param})

      files = []
      archives = []

      for resource in pig_script.dict['resources']:
        if resource['type'] == 'file':
          files.append(resource['value'])
        if resource['type'] == 'archive':
          archives.append({"dummy": "", "name": resource['value']})

      action = Pig.objects.create(
          name='pig',
          script_path=script_path,
          workflow=workflow,
          node_type='pig',
          params=json.dumps(pig_params),
          files=json.dumps(files),
          archives=json.dumps(archives),
      )

      action.add_node(workflow.end)

      start_link = workflow.start.get_link()
      start_link.child = action
      start_link.save()

      oozie_wf = _submit_workflow(self.user, self.fs, workflow, mapping)
    finally:
      workflow.delete()

    return oozie_wf
Beispiel #4
0
  def submit(self, pig_script, params):
    mapping = {
      'oozie.use.system.libpath':  'true',
    }

    workflow = None

    try:
      workflow = self._create_workflow(pig_script, params)
      oozie_wf = _submit_workflow(self.user, self.fs, workflow, mapping)
    finally:
      if workflow:
        workflow.delete()

    return oozie_wf
Beispiel #5
0
    def submit(self, pig_script, params):
        mapping = {
            'oozie.use.system.libpath': 'true',
        }

        workflow = None

        try:
            workflow = self._create_workflow(pig_script, params)
            oozie_wf = _submit_workflow(self.user, self.fs, workflow, mapping)
        finally:
            if workflow:
                workflow.delete()

        return oozie_wf