예제 #1
0
 def _pick_spark_tmp_dir(self):
     if self._opts['spark_tmp_dir']:
         return self.fs.join(self._opts['spark_tmp_dir'], self._job_key)
     else:
         master = self._spark_master() or 'local'
         if master.startswith('local'):  # including local-cluster
             # need a local temp dir
             # add "-spark" so we don't collide with default local temp dir
             return os.path.join(gettempdir(), self._job_key + '-spark')
         else:
             # use HDFS (same default as HadoopJobRunner)
             return posixpath.join(fully_qualify_hdfs_path('tmp/mrjob'),
                                   self._job_key)
예제 #2
0
파일: runner.py 프로젝트: Yelp/mrjob
 def _pick_spark_tmp_dir(self):
     if self._opts['spark_tmp_dir']:
         return self.fs.join(self._opts['spark_tmp_dir'], self._job_key)
     else:
         master = self._spark_master() or 'local'
         if master.startswith('local'):  # including local-cluster
             # need a local temp dir
             # add "-spark" so we don't collide with default local temp dir
             return os.path.join(
                 gettempdir(), self._job_key + '-spark')
         else:
             # use HDFS (same default as HadoopJobRunner)
             return posixpath.join(
                 fully_qualify_hdfs_path('tmp/mrjob'), self._job_key)
예제 #3
0
 def _pick_spark_tmp_dir(self):
     if self._opts['spark_tmp_dir']:
         if is_uri(self._opts['spark_tmp_dir']):
             return posixpath.join(
                 self._opts['spark_tmp_dir'], self._job_key)
         else:
             return os.path.join(
                 self._opts['spark_tmp_dir'], self._job_key)
     elif self._spark_master_is_local():
         # need a local temp dir
         # add "-spark" so we don't collide with default local temp dir
         return os.path.join(
             gettempdir(), self._job_key + '-spark')
     else:
         # use HDFS (same default as HadoopJobRunner)
         return posixpath.join(
             fully_qualify_hdfs_path('tmp/mrjob'), self._job_key)
예제 #4
0
 def test_hdfs_uri(self):
     self.assertEqual(fully_qualify_hdfs_path('hdfs://host/path/'),
                      'hdfs://host/path/')
예제 #5
0
 def test_absolute_path(self):
     self.assertEqual(fully_qualify_hdfs_path('/path/to/cheese'),
                      'hdfs:///path/to/cheese')
예제 #6
0
 def test_relative_path(self):
     with patch('getpass.getuser') as getuser:
         getuser.return_value = 'dave'
         self.assertEqual(fully_qualify_hdfs_path('path/to/chocolate'),
                          'hdfs:///user/dave/path/to/chocolate')
예제 #7
0
 def test_empty(self):
     with patch('getpass.getuser') as getuser:
         getuser.return_value = 'dave'
         self.assertEqual(fully_qualify_hdfs_path(''), 'hdfs:///user/dave/')
예제 #8
0
파일: test_hadoop.py 프로젝트: pyzen/mrjob
 def test_other_uri(self):
     self.assertEqual(fully_qualify_hdfs_path('foo://bar/baz'),
                      'foo://bar/baz')
예제 #9
0
파일: test_hadoop.py 프로젝트: pyzen/mrjob
 def test_s3n_uri(self):
     self.assertEqual(fully_qualify_hdfs_path('s3n://bucket/oh/noes'),
                      's3n://bucket/oh/noes')
예제 #10
0
파일: test_hadoop.py 프로젝트: pyzen/mrjob
 def test_absolute_path(self):
     self.assertEqual(fully_qualify_hdfs_path('/path/to/cheese'),
                      'hdfs:///path/to/cheese')
예제 #11
0
파일: test_hadoop.py 프로젝트: pyzen/mrjob
 def test_relative_path(self):
     with patch('getpass.getuser') as getuser:
         getuser.return_value = 'dave'
         self.assertEqual(fully_qualify_hdfs_path('path/to/chocolate'),
                          'hdfs:///user/dave/path/to/chocolate')
예제 #12
0
파일: test_hadoop.py 프로젝트: pyzen/mrjob
 def test_empty(self):
     with patch('getpass.getuser') as getuser:
         getuser.return_value = 'dave'
         self.assertEqual(fully_qualify_hdfs_path(''), 'hdfs:///user/dave/')
예제 #13
0
 def test_relative_path(self):
     with patch("getpass.getuser") as getuser:
         getuser.return_value = "dave"
         self.assertEqual(fully_qualify_hdfs_path("path/to/chocolate"), "hdfs:///user/dave/path/to/chocolate")
예제 #14
0
 def test_empty(self):
     with patch("getpass.getuser") as getuser:
         getuser.return_value = "dave"
         self.assertEqual(fully_qualify_hdfs_path(""), "hdfs:///user/dave/")
예제 #15
0
 def test_s3n_uri(self):
     self.assertEqual(fully_qualify_hdfs_path('s3n://bucket/oh/noes'),
                      's3n://bucket/oh/noes')
예제 #16
0
 def test_other_uri(self):
     self.assertEqual(fully_qualify_hdfs_path('foo://bar/baz'),
                      'foo://bar/baz')
예제 #17
0
파일: test_hadoop.py 프로젝트: pyzen/mrjob
 def test_hdfs_uri(self):
     self.assertEqual(fully_qualify_hdfs_path('hdfs://host/path/'),
                      'hdfs://host/path/')