def _step_output_uri(self, step_num): """URI to use as output for the given step. This is either an intermediate dir (see :py:meth:`intermediate_output_uri`) or ``self._output_dir`` for the final step.""" if step_num == len(self._get_steps()) - 1: return to_uri(self._output_dir) else: return to_uri(self._intermediate_output_dir(step_num))
def _step_output_uri(self, step_num): """URI to use as output for the given step. This is either an intermediate dir (see :py:meth:`intermediate_output_uri`) or ``self._output_dir`` for the final step.""" if step_num == len(self._get_steps()) - 1: return to_uri(self._output_dir) else: return to_uri(self._intermediate_output_dir(step_num))
def _step_input_uris(self, step_num): """A list of URIs to use as input for the given step. For all except the first step, this list will have a single item (a directory).""" if step_num == 0: return [self._upload_mgr.uri(path) if self._upload_mgr else to_uri(path) for path in self._get_input_paths()] else: return [to_uri(self._intermediate_output_dir(step_num - 1))]
def _step_input_uris(self, step_num): """A list of URIs to use as input for the given step. For all except the first step, this list will have a single item (a directory).""" if step_num == 0: return [ self._upload_mgr.uri(path) if self._upload_mgr else to_uri(path) for path in self._get_input_paths() ] else: return [to_uri(self._intermediate_output_dir(step_num - 1))]
def test_relative_path_to_uri(self): tmp_dir = realpath(gettempdir()) with save_cwd(): chdir(tmp_dir) foo_uri = to_uri('foo.db') self.assertEqual(foo_uri[:8], 'file:///') self.assertEqual(foo_uri, 'file://' + join(pathname2url(tmp_dir), 'foo.db'))
def _check_spark_tmp_dir_opt(self): # warn if spark_tmp_dir isn't actually visible to Spark executors # (see #2062) tmp_dir_is_local = to_uri( self._opts['spark_tmp_dir']).startswith('file://') spark_master_is_local = self._spark_master().startswith('local') if tmp_dir_is_local != spark_master_is_local: log.warning( 'Warning: executors on Spark master %s may not be able to' ' access spark_tmp_dir %s' % (self._spark_master(), self._opts['spark_tmp_dir']))
def test_local_output_dir_and_step_output_dir(self): input1_path = self.makefile('input1') input2_path = self.makefile('input2') output_dir = self.makedirs('output') step_output_dir = self.makedirs('step_output') # this has three steps, which lets us test step numbering job = MRCountingJob([ '-r', 'local', '--output-dir', output_dir, '--step-output-dir', step_output_dir, input1_path, input2_path]) job.sandbox() with job.make_runner() as runner: self.assertEqual(runner._num_steps(), 3) input_uris_0 = runner._step_input_uris(0) self.assertEqual([os.path.basename(uri) for uri in input_uris_0], ['input1', 'input2']) self.assertEqual([uri[:8] for uri in input_uris_0], ['file:///', 'file:///']) output_uri_0 = runner._step_output_uri(0) self.assertEqual(output_uri_0, to_uri(os.path.join(step_output_dir, '0000'))) input_uris_1 = runner._step_input_uris(1) self.assertEqual(input_uris_1, [output_uri_0]) output_uri_1 = runner._step_output_uri(1) self.assertEqual(output_uri_1, to_uri(os.path.join(step_output_dir, '0001'))) input_uris_2 = runner._step_input_uris(2) self.assertEqual(input_uris_2, [output_uri_1]) output_uri_2 = runner._step_output_uri(2) self.assertEqual(output_uri_2, to_uri(output_dir))
def test_local_output_dir_and_step_output_dir(self): input1_path = self.makefile('input1') input2_path = self.makefile('input2') output_dir = self.makedirs('output') step_output_dir = self.makedirs('step_output') # this has three steps, which lets us test step numbering job = MRCountingJob([ '-r', 'local', '--output-dir', output_dir, '--step-output-dir', step_output_dir, input1_path, input2_path]) job.sandbox() with job.make_runner() as runner: self.assertEqual(runner._num_steps(), 3) input_uris_0 = runner._step_input_uris(0) self.assertEqual([os.path.basename(uri) for uri in input_uris_0], ['input1', 'input2']) self.assertEqual([uri[:8] for uri in input_uris_0], ['file:///', 'file:///']) output_uri_0 = runner._step_output_uri(0) self.assertEqual(output_uri_0, to_uri(os.path.join(step_output_dir, '0000'))) input_uris_1 = runner._step_input_uris(1) self.assertEqual(input_uris_1, [output_uri_0]) output_uri_1 = runner._step_output_uri(1) self.assertEqual(output_uri_1, to_uri(os.path.join(step_output_dir, '0001'))) input_uris_2 = runner._step_input_uris(2) self.assertEqual(input_uris_2, [output_uri_1]) output_uri_2 = runner._step_output_uri(2) self.assertEqual(output_uri_2, to_uri(output_dir))
def test_to_uri(self): self.assertEqual(to_uri('/path/to/file'), 'file:///path/to/file') self.assertEqual(to_uri('s3://a/uri'), 's3://a/uri')