def stat_on_local(self):
wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR)
p_ = os.path.join(wd_, make_random_str())
if hdfs.default_is_local():
wd, p = wd_, p_
host = "default"
else:
wd, p = ('file:%s' % _ for _ in (wd_, p_))
host = ""
fs = hdfs.hdfs(host, 0)
with fs.open_file(p_, 'w') as fo:
fo.write(make_random_str())
info = fs.get_path_info(p_)
fs.close()
s = hdfs.path.stat(p)
os_s = os.stat(p_)
for n in dir(s):
if n.startswith('st_'):
try:
exp_v = getattr(os_s, n)
except AttributeError:
try:
exp_v = info[self.NMAP[n]]
except KeyError:
continue
self.assertEqual(getattr(s, n), exp_v)
self.__check_extra_args(s, info)
self.__check_wrapper_funcs(p)
hdfs.rmr(wd)
def stat_on_local(self):
wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR)
p_ = os.path.join(wd_, make_random_str())
if hdfs.default_is_local():
wd, p = wd_, p_
host = "default"
else:
wd, p = ('file:%s' % _ for _ in (wd_, p_))
host = ""
fs = hdfs.hdfs(host, 0)
with fs.open_file(p_, 'w') as fo:
fo.write(make_random_str())
info = fs.get_path_info(p_)
fs.close()
s = hdfs.path.stat(p)
os_s = os.stat(p_)
for n in dir(s):
if n.startswith('st_'):
try:
exp_v = getattr(os_s, n)
except AttributeError:
try:
exp_v = info[self.NMAP[n]]
except KeyError:
continue
self.assertEqual(getattr(s, n), exp_v)
self.__check_extra_args(s, info)
self.__check_wrapper_funcs(p)
hdfs.rmr(wd)
def good(self):
base_path = make_random_str()
for path in base_path, base_path + UNI_CHR:
hdfs.dump("foo\n", path)
self.assertTrue(hdfs.path.exists(path))
hdfs.rmr(path)
self.assertFalse(hdfs.path.exists(path))
def samefile_link(self):
wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR)
wd = 'file:%s' % wd_
link = os.path.join(wd_, make_random_str())
os.symlink(wd_, link)
self.assertTrue(hdfs.path.samefile('file:%s' % link, 'file:%s' % wd_))
hdfs.rmr(wd)
def samefile_link(self):
wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR)
wd = 'file:%s' % wd_
link = os.path.join(wd_, make_random_str())
os.symlink(wd_, link)
self.assertTrue(hdfs.path.samefile('file:%s' % link, 'file:%s' % wd_))
hdfs.rmr(wd)
def good(self):
base_path = make_random_str()
for path in base_path, base_path + UNI_CHR:
hdfs.dump("foo\n", path)
self.assertTrue(hdfs.path.exists(path))
hdfs.rmr(path)
self.assertFalse(hdfs.path.exists(path))
def __init__(self, prefix=None, logger=None):
self.wd = self.exe = self.input = self.output = None
self.logger = logger or utils.NullLogger()
if prefix:
self.wd = utils.make_random_str(prefix=prefix)
hdfs.mkdir(self.wd)
for n in "input", "output":
setattr(self, n, hdfs.path.join(self.wd, n))
def realpath(self):
wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR)
wd = 'file:%s' % wd_
link = os.path.join(wd_, make_random_str())
os.symlink(wd_, link)
expected_path = 'file:%s' % os.path.realpath(wd_)
self.assertEqual(hdfs.path.realpath('file:%s' % link), expected_path)
hdfs.rmr(wd)
def __init__(self, prefix=None, logger=None):
self.wd = self.exe = self.input = self.output = None
self.logger = logger or utils.NullLogger()
if prefix:
self.wd = utils.make_random_str(prefix=prefix)
hdfs.mkdir(self.wd)
for n in "input", "output":
setattr(self, n, hdfs.path.join(self.wd, n))
def realpath(self):
wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR)
wd = 'file:%s' % wd_
link = os.path.join(wd_, make_random_str())
os.symlink(wd_, link)
expected_path = 'file:%s' % os.path.realpath(wd_)
self.assertEqual(hdfs.path.realpath('file:%s' % link), expected_path)
hdfs.rmr(wd)
def set_exe(self, pipes_code):
"""
Dump launcher code to the distributed file system.
"""
if not self.output:
raise RuntimeError("no output directory, can't create launcher")
parent = hdfs.path.dirname(hdfs.path.abspath(self.output.rstrip("/")))
self.exe = hdfs.path.join(parent, utils.make_random_str())
hdfs.dump(pipes_code, self.exe)
def realpath(self):
wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR)
wd = 'file:%s' % wd_
link = os.path.join(wd_, make_random_str())
os.symlink(wd_, link)
expected_path = ('file:%s%s' % ("/private", wd_)
if sys.platform == "darwin" else 'file:%s' % wd_)
self.assertEqual(hdfs.path.realpath('file:%s' % link), expected_path)
hdfs.rmr(wd)
def set_exe(self, pipes_code):
"""
Dump launcher code to the distributed file system.
"""
if not self.output:
raise RuntimeError("no output directory, can't create launcher")
parent = hdfs.path.dirname(hdfs.path.abspath(self.output.rstrip("/")))
self.exe = hdfs.path.join(parent, utils.make_random_str())
hdfs.dump(pipes_code, self.exe)
def stat(self):
if hdfs.default_is_local():
return
bn = '%s%s' % (make_random_str(), UNI_CHR)
fn = '/user/%s/%s' % (DEFAULT_USER, bn)
fs = hdfs.hdfs("default", 0)
p = "hdfs://%s:%s%s" % (fs.host, fs.port, fn)
with fs.open_file(fn, 'w') as fo:
fo.write(make_random_str())
info = fs.get_path_info(fn)
fs.close()
s = hdfs.path.stat(p)
for n1, n2 in self.NMAP.iteritems():
attr = getattr(s, n1, None)
self.assertFalse(attr is None)
self.assertEqual(attr, info[n2])
self.__check_extra_args(s, info)
self.__check_wrapper_funcs(p)
hdfs.rmr(p)
def realpath(self):
wd_ = tempfile.mkdtemp(prefix='pydoop_', suffix=UNI_CHR)
wd = 'file:%s' % wd_
link = os.path.join(wd_, make_random_str())
os.symlink(wd_, link)
expected_path = ('file:%s%s' % ("/private", wd_)
if sys.platform == "darwin"
else 'file:%s' % wd_)
self.assertEqual(hdfs.path.realpath('file:%s' % link), expected_path)
hdfs.rmr(wd)
def stat(self):
if hdfs.default_is_local():
return
bn = '%s%s' % (make_random_str(), UNI_CHR)
fn = '/user/%s/%s' % (DEFAULT_USER, bn)
fs = hdfs.hdfs("default", 0)
p = "hdfs://%s:%s%s" % (fs.host, fs.port, fn)
with fs.open_file(fn, 'w') as fo:
fo.write(make_random_str())
info = fs.get_path_info(fn)
fs.close()
s = hdfs.path.stat(p)
for n1, n2 in self.NMAP.iteritems():
attr = getattr(s, n1, None)
self.assertFalse(attr is None)
self.assertEqual(attr, info[n2])
self.__check_extra_args(s, info)
self.__check_wrapper_funcs(p)
hdfs.rmr(p)
def runTest(self):
path = make_random_str() + UNI_CHR
hdfs.dump("foo\n", path)
st = hdfs.path.stat(path)
atime, mtime = [getattr(st, 'st_%stime' % _) for _ in 'am']
new_atime, new_mtime = atime + 100, mtime + 200
hdfs.path.utime(path, (new_atime, new_mtime))
st = hdfs.path.stat(path)
self.assertEqual(st.st_atime, new_atime)
self.assertEqual(st.st_mtime, new_mtime)
hdfs.rmr(path)
def runTest(self):
path = make_random_str() + UNI_CHR
hdfs.dump("foo\n", path)
st = hdfs.path.stat(path)
atime, mtime = [getattr(st, 'st_%stime' % _) for _ in 'am']
new_atime, new_mtime = atime + 100, mtime + 200
hdfs.path.utime(path, (new_atime, new_mtime))
st = hdfs.path.stat(path)
self.assertEqual(st.st_atime, new_atime)
self.assertEqual(st.st_mtime, new_mtime)
hdfs.rmr(path)
def __init__(self, prefix=None, logger=None):
hadoop_version_info = pydoop.hadoop_version_info()
if hadoop_version_info.is_local():
raise pydoop.LocalModeNotSupported()
self.wd = self.exe = self.input = self.output = None
self.logger = logger or utils.NullLogger()
if prefix:
self.wd = utils.make_random_str(prefix=prefix)
hdfs.mkdir(self.wd)
for n in "input", "output":
setattr(self, n, hdfs.path.join(self.wd, n))
def __init__(self, prefix=None, logger=None):
hadoop_version_info = pydoop.hadoop_version_info()
if hadoop_version_info.is_local():
raise pydoop.LocalModeNotSupported()
self.wd = self.exe = self.input = self.output = None
self.logger = logger or utils.NullLogger()
if prefix:
self.wd = utils.make_random_str(prefix=prefix)
hdfs.mkdir(self.wd)
for n in "input", "output":
setattr(self, n, hdfs.path.join(self.wd, n))
def setUp(self):
self.path = make_random_str() + UNI_CHR
hdfs.dump("foo\n", self.path)
def test_other(self):
self.__test(0, user=make_random_str())
def setUp(self):
self.path = make_random_str() + UNI_CHR
hdfs.mkdir(self.path)
def samefile_rel(self):
p = make_random_str() + UNI_CHR
hdfs.dump("foo\n", p)
self.assertTrue(hdfs.path.samefile(p, hdfs.path.abspath(p)))
hdfs.rmr(p)
def setUp(self):
self.path = make_random_str()
self.u_path = self.path + UNI_CHR
def samefile_rel(self):
p = make_random_str() + UNI_CHR
hdfs.dump("foo\n", p)
self.assertTrue(hdfs.path.samefile(p, hdfs.path.abspath(p)))
hdfs.rmr(p)
def setUp(self):
self.path = make_random_str()
self.u_path = self.path + UNI_CHR
def setUp(self):
self.path = make_random_str() + UNI_CHR
hdfs.mkdir(self.path)
def test_other(self):
self.__test(0, user=make_random_str())
def setUp(self):
self.path = make_random_str() + UNI_CHR
hdfs.dump("foo\n", self.path)
