Example #1
0
    def mk_task_results(self, task_name, mode='map', host="localhost"):
        """
    Creates a file suitable for using as task results and return it's url
    """

        job_home = disco.job_home(self.job_name,
                                  os.path.join(self.data_root, host))

        self.mk_output_file('{}-0'.format(mode), 'line1\n'
                            'line2\n',
                            job_home=job_home)

        self.mk_output_file('{}-1'.format(mode), 'line1\n'
                            'line2\n',
                            job_home=job_home)

        self.mk_output_file('{}-2'.format(mode), 'line1\n'
                            'line2\n',
                            job_home=job_home)

        job_url = disco.job_url(host, self.job_name)

        makedirs(job_home)
        task_result_path = os.path.join(job_home, task_name)

        cat(task_result_path,
            ("0  part://{host}/{job_url}/{mode}-0\n"
             "1  part://{host}/{job_url}/{mode}-1\n"
             "0  part://{host}/{job_url}/{mode}-2\n").format(job_url=job_url,
                                                             host=host,
                                                             mode=mode))

        return os.path.join("disco://", host, job_url, task_name)
Example #2
0
  def setUp(self):

    self.data_root = tempfile.mkdtemp()

    self.job_name = "Job@123"
    self.host = "localhost"
    self.job_home = disco.job_home(self.job_name, os.path.join(self.data_root, self.host))
    self.job_url = disco.job_url(self.host, self.job_name)

    makedirs(self.job_home)
 
    self.part_info = self.make_part_info(self.job_home)
Example #3
0
    def setUp(self):

        self.data_root = tempfile.mkdtemp()

        self.job_name = "Job@123"
        self.host = "localhost"
        self.job_home = disco.job_home(self.job_name,
                                       os.path.join(self.data_root, self.host))
        self.job_url = disco.job_url(self.host, self.job_name)

        makedirs(self.job_home)

        self.part_info = self.make_part_info(self.job_home)
Example #4
0
  def mk_task_results(self, task_name, mode='map', host="localhost"):
    """
    Creates a file suitable for using as task results and return it's url
    """

    job_home = disco.job_home(self.job_name, os.path.join(self.data_root, host))

    self.mk_output_file('{}-0'.format(mode), 
      'line1\n'
      'line2\n',
      job_home=job_home
    )

    self.mk_output_file('{}-1'.format(mode), 
      'line1\n'
      'line2\n',
      job_home=job_home
    )

    self.mk_output_file('{}-2'.format(mode), 
      'line1\n'
      'line2\n',
      job_home=job_home
    )


  

    job_url = disco.job_url(host, self.job_name)

    makedirs(job_home)
    task_result_path = os.path.join(job_home, task_name)

    cat(task_result_path, 
      (
        "0  part://{host}/{job_url}/{mode}-0\n"
        "1  part://{host}/{job_url}/{mode}-1\n"
        "0  part://{host}/{job_url}/{mode}-2\n"
      ).format(job_url = job_url, host=host, mode=mode)
    )

    return os.path.join("disco://", host, job_url, task_name)