def stat_on_local(self): wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR) p_ = os.path.join(wd_, make_random_str()) if hdfs.default_is_local(): wd, p = wd_, p_ host = "default" else: wd, p = ('file:%s' % _ for _ in (wd_, p_)) host = "" fs = hdfs.hdfs(host, 0) with fs.open_file(p_, 'w') as fo: fo.write(make_random_str()) info = fs.get_path_info(p_) fs.close() s = hdfs.path.stat(p) os_s = os.stat(p_) for n in dir(s): if n.startswith('st_'): try: exp_v = getattr(os_s, n) except AttributeError: try: exp_v = info[self.NMAP[n]] except KeyError: continue self.assertEqual(getattr(s, n), exp_v) self.__check_extra_args(s, info) self.__check_wrapper_funcs(p) hdfs.rmr(wd)
def good(self): base_path = make_random_str() for path in base_path, base_path + UNI_CHR: hdfs.dump("foo\n", path) self.assertTrue(hdfs.path.exists(path)) hdfs.rmr(path) self.assertFalse(hdfs.path.exists(path))
def samefile_link(self): wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR) wd = 'file:%s' % wd_ link = os.path.join(wd_, make_random_str()) os.symlink(wd_, link) self.assertTrue(hdfs.path.samefile('file:%s' % link, 'file:%s' % wd_)) hdfs.rmr(wd)
def __init__(self, prefix=None, logger=None): self.wd = self.exe = self.input = self.output = None self.logger = logger or utils.NullLogger() if prefix: self.wd = utils.make_random_str(prefix=prefix) hdfs.mkdir(self.wd) for n in "input", "output": setattr(self, n, hdfs.path.join(self.wd, n))
def realpath(self): wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR) wd = 'file:%s' % wd_ link = os.path.join(wd_, make_random_str()) os.symlink(wd_, link) expected_path = 'file:%s' % os.path.realpath(wd_) self.assertEqual(hdfs.path.realpath('file:%s' % link), expected_path) hdfs.rmr(wd)
def set_exe(self, pipes_code): """ Dump launcher code to the distributed file system. """ if not self.output: raise RuntimeError("no output directory, can't create launcher") parent = hdfs.path.dirname(hdfs.path.abspath(self.output.rstrip("/"))) self.exe = hdfs.path.join(parent, utils.make_random_str()) hdfs.dump(pipes_code, self.exe)
def realpath(self): wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR) wd = 'file:%s' % wd_ link = os.path.join(wd_, make_random_str()) os.symlink(wd_, link) expected_path = ('file:%s%s' % ("/private", wd_) if sys.platform == "darwin" else 'file:%s' % wd_) self.assertEqual(hdfs.path.realpath('file:%s' % link), expected_path) hdfs.rmr(wd)
def stat(self): if hdfs.default_is_local(): return bn = '%s%s' % (make_random_str(), UNI_CHR) fn = '/user/%s/%s' % (DEFAULT_USER, bn) fs = hdfs.hdfs("default", 0) p = "hdfs://%s:%s%s" % (fs.host, fs.port, fn) with fs.open_file(fn, 'w') as fo: fo.write(make_random_str()) info = fs.get_path_info(fn) fs.close() s = hdfs.path.stat(p) for n1, n2 in self.NMAP.iteritems(): attr = getattr(s, n1, None) self.assertFalse(attr is None) self.assertEqual(attr, info[n2]) self.__check_extra_args(s, info) self.__check_wrapper_funcs(p) hdfs.rmr(p)
def runTest(self): path = make_random_str() + UNI_CHR hdfs.dump("foo\n", path) st = hdfs.path.stat(path) atime, mtime = [getattr(st, 'st_%stime' % _) for _ in 'am'] new_atime, new_mtime = atime + 100, mtime + 200 hdfs.path.utime(path, (new_atime, new_mtime)) st = hdfs.path.stat(path) self.assertEqual(st.st_atime, new_atime) self.assertEqual(st.st_mtime, new_mtime) hdfs.rmr(path)
def __init__(self, prefix=None, logger=None): hadoop_version_info = pydoop.hadoop_version_info() if hadoop_version_info.is_local(): raise pydoop.LocalModeNotSupported() self.wd = self.exe = self.input = self.output = None self.logger = logger or utils.NullLogger() if prefix: self.wd = utils.make_random_str(prefix=prefix) hdfs.mkdir(self.wd) for n in "input", "output": setattr(self, n, hdfs.path.join(self.wd, n))
def setUp(self): self.path = make_random_str() + UNI_CHR hdfs.dump("foo\n", self.path)
def test_other(self): self.__test(0, user=make_random_str())
def setUp(self): self.path = make_random_str() + UNI_CHR hdfs.mkdir(self.path)
def samefile_rel(self): p = make_random_str() + UNI_CHR hdfs.dump("foo\n", p) self.assertTrue(hdfs.path.samefile(p, hdfs.path.abspath(p))) hdfs.rmr(p)
def setUp(self): self.path = make_random_str() self.u_path = self.path + UNI_CHR