Exemplo n.º 1
0
    def _upload_workspace(self, morphline):
        hdfs_workspace_path = Job.get_workspace(self.username)
        hdfs_morphline_path = os.path.join(hdfs_workspace_path,
                                           "morphline.conf")
        hdfs_workflow_path = os.path.join(hdfs_workspace_path, "workflow.xml")
        hdfs_log4j_properties_path = os.path.join(hdfs_workspace_path,
                                                  "log4j.properties")

        workflow_template_path = os.path.join(
            CONFIG_INDEXING_TEMPLATES_PATH.get(), "workflow.xml")
        log4j_template_path = os.path.join(
            CONFIG_INDEXING_TEMPLATES_PATH.get(), "log4j.properties")

        # create workspace on hdfs
        self.fs.do_as_user(self.username, self.fs.mkdir, hdfs_workspace_path)

        self.fs.do_as_user(self.username,
                           self.fs.create,
                           hdfs_morphline_path,
                           data=morphline)
        self.fs.do_as_user(self.username,
                           self.fs.create,
                           hdfs_workflow_path,
                           data=open(workflow_template_path).read())
        self.fs.do_as_user(self.username,
                           self.fs.create,
                           hdfs_log4j_properties_path,
                           data=open(log4j_template_path).read())

        return hdfs_workspace_path
Exemplo n.º 2
0
  def _upload_workspace(self, configs):
    from oozie.models2 import Job

    hdfs_workspace_path = Job.get_workspace(self.username)

    # Create workspace on hdfs
    self.fs.do_as_user(self.username, self.fs.mkdir, hdfs_workspace_path)

    for config_name, config_content in configs.items():
      hdfs_config_path = os.path.join(hdfs_workspace_path, config_name)
      self.fs.do_as_user(self.username, self.fs.create, hdfs_config_path, data=config_content)

    return hdfs_workspace_path
Exemplo n.º 3
0
  def _upload_workspace(self, morphline):
    hdfs_workspace_path = Job.get_workspace(self.username)
    hdfs_morphline_path = os.path.join(hdfs_workspace_path, "morphline.conf")
    hdfs_log4j_properties_path = os.path.join(hdfs_workspace_path, "log4j.properties")

    log4j_template_path = os.path.join(CONFIG_INDEXING_TEMPLATES_PATH.get(), "log4j.properties")

    # Create workspace on hdfs
    self.fs.do_as_user(self.username, self.fs.mkdir, hdfs_workspace_path)

    self.fs.do_as_user(self.username, self.fs.create, hdfs_morphline_path, data=morphline)
    self.fs.do_as_user(self.username, self.fs.create, hdfs_log4j_properties_path, data=open(log4j_template_path).read())

    return hdfs_workspace_path
Exemplo n.º 4
0
    def _upload_workspace(self, envelope):
        from oozie.models2 import Job

        hdfs_workspace_path = Job.get_workspace(self.username)
        hdfs_envelope_path = os.path.join(hdfs_workspace_path, "envelope.conf")

        # Create workspace on hdfs
        self.fs.do_as_user(self.username, self.fs.mkdir, hdfs_workspace_path)

        self.fs.do_as_user(self.username,
                           self.fs.create,
                           hdfs_envelope_path,
                           data=envelope)

        return hdfs_workspace_path