def test_process_url_local(self):
self.mk_output_file('map-0', 'line1\n' 'line2\n')
self.mk_output_file('map-1', 'line3\n' 'line4\n')
part_path, part_url = self.part_info
part_dir = os.path.basename(part_path)
eq_(
'0 disco://localhost/{}/{}/part-0\n'.format(
self.job_url, part_dir),
shuffle.process_url(
("0", "part://localhost/{}/map-0".format(self.job_url)),
self.data_root, self.part_info))
eq_(
open(os.path.join(part_path, "part-0")).read(), 'line1\n'
'line2\n')
eq_(
'0 disco://localhost/{}/{}/part-0\n'.format(
self.job_url, part_dir),
shuffle.process_url(
("0", "part://localhost/{}/map-1".format(self.job_url)),
self.data_root, self.part_info))
eq_(
open(os.path.join(part_path, "part-0")).read(), 'line1\n'
'line2\n'
'line3\n'
'line4\n')
def test_process_url_local(self):
self.mk_output_file('map-0',
'line1\n'
'line2\n'
)
self.mk_output_file('map-1',
'line3\n'
'line4\n'
)
part_path,part_url = self.part_info
part_dir = os.path.basename(part_path)
eq_(
'0 disco://localhost/{}/{}/part-0\n'.format(self.job_url, part_dir),
shuffle.process_url(
("0", "part://localhost/{}/map-0".format(self.job_url)),
self.data_root,
self.part_info
)
)
eq_(
open(os.path.join(part_path, "part-0")).read(),
'line1\n'
'line2\n'
)
eq_(
'0 disco://localhost/{}/{}/part-0\n'.format(self.job_url, part_dir),
shuffle.process_url(
("0", "part://localhost/{}/map-1".format(self.job_url)),
self.data_root,
self.part_info
)
)
eq_(
open(os.path.join(part_path, "part-0")).read(),
'line1\n'
'line2\n'
'line3\n'
'line4\n'
)
def test_process_url_non_local(self):
eq_(
'0 tag://blah\n',
shuffle.process_url(
("0", "tag://blah"),
self.data_root,
self.part_info
)
)
def test_process_url_non_local(self):
eq_(
'0 tag://blah\n',
shuffle.process_url(("0", "tag://blah"), self.data_root,
self.part_info))
