def mk_task_results(self, task_name, mode='map', host="localhost"):
"""
Creates a file suitable for using as task results and return it's url
"""
job_home = disco.job_home(self.job_name,
os.path.join(self.data_root, host))
self.mk_output_file('{}-0'.format(mode), 'line1\n'
'line2\n',
job_home=job_home)
self.mk_output_file('{}-1'.format(mode), 'line1\n'
'line2\n',
job_home=job_home)
self.mk_output_file('{}-2'.format(mode), 'line1\n'
'line2\n',
job_home=job_home)
job_url = disco.job_url(host, self.job_name)
makedirs(job_home)
task_result_path = os.path.join(job_home, task_name)
cat(task_result_path,
("0 part://{host}/{job_url}/{mode}-0\n"
"1 part://{host}/{job_url}/{mode}-1\n"
"0 part://{host}/{job_url}/{mode}-2\n").format(job_url=job_url,
host=host,
mode=mode))
return os.path.join("disco://", host, job_url, task_name)
def make_part_info(self, job_home):
part_dir = "partitions-{}".format(disco.timestamp())
part_path = os.path.join(job_home, part_dir)
makedirs(part_path)
part_url = os.path.join("disco://localhost", self.job_url, part_dir)
return (part_path, part_url)
def setUp(self):
self.data_root = tempfile.mkdtemp()
self.job_name = "Job@123"
self.host = "localhost"
self.job_home = disco.job_home(self.job_name, os.path.join(self.data_root, self.host))
self.job_url = disco.job_url(self.host, self.job_name)
makedirs(self.job_home)
self.part_info = self.make_part_info(self.job_home)
def setUp(self):
self.data_root = tempfile.mkdtemp()
self.job_name = "Job@123"
self.host = "localhost"
self.job_home = disco.job_home(self.job_name,
os.path.join(self.data_root, self.host))
self.job_url = disco.job_url(self.host, self.job_name)
makedirs(self.job_home)
self.part_info = self.make_part_info(self.job_home)
def make_part_info(self, job_home):
part_dir = "partitions-{}".format(disco.timestamp())
part_path = os.path.join(
job_home,
part_dir
)
makedirs(part_path)
part_url = os.path.join("disco://localhost", self.job_url, part_dir)
return (
part_path,
part_url
)
def mk_task_results(self, task_name, mode='map', host="localhost"):
"""
Creates a file suitable for using as task results and return it's url
"""
job_home = disco.job_home(self.job_name, os.path.join(self.data_root, host))
self.mk_output_file('{}-0'.format(mode),
'line1\n'
'line2\n',
job_home=job_home
)
self.mk_output_file('{}-1'.format(mode),
'line1\n'
'line2\n',
job_home=job_home
)
self.mk_output_file('{}-2'.format(mode),
'line1\n'
'line2\n',
job_home=job_home
)
job_url = disco.job_url(host, self.job_name)
makedirs(job_home)
task_result_path = os.path.join(job_home, task_name)
cat(task_result_path,
(
"0 part://{host}/{job_url}/{mode}-0\n"
"1 part://{host}/{job_url}/{mode}-1\n"
"0 part://{host}/{job_url}/{mode}-2\n"
).format(job_url = job_url, host=host, mode=mode)
)
return os.path.join("disco://", host, job_url, task_name)