def _iterDataSorc(self, path, info):
if not os.path.isabs(path):
basedir = self.get('basedir')
if basedir:
path = os.path.join(basedir, path)
onfo = info.get('open')
for fd in s_filepath.openfiles(path, mode='rb'):
yield iterdata(fd, **onfo)
def diropen(path):
[f for f in s_filepath.openfiles(path, mode='rb', req=True)]
def test_filepath_glob(self):
temp_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(temp_dir, 'dir0'))
os.mkdir(os.path.join(temp_dir, 'dir1'))
os.mkdir(os.path.join(temp_dir, 'dir2'))
os.mkdir(os.path.join(temp_dir, 'fooD'))
f0 = b'A'*20
f0_path = os.path.join(temp_dir, 'foo0')
with open(f0_path, 'wb') as fd:
fd.write(f0)
f1 = b'B'*20
f1_path = os.path.join(temp_dir, 'foo1')
with open(f1_path, 'wb') as fd:
fd.write(f1)
f2 = b'C'*20
f2_path = os.path.join(temp_dir, 'foo2')
with open(f2_path, 'wb') as fd:
fd.write(f2)
f3 = b'Z'*20
f3_path = os.path.join(temp_dir, 'junk')
with open(f3_path, 'wb') as fd:
fd.write(f3)
# files that exists
path = os.path.join(temp_dir, 'foo*')
self.assertFalse(s_filepath.exists(path))
self.assertFalse(s_filepath.isfile(path))
self.assertFalse(s_filepath.isdir(path))
# dirs that exist
path = os.path.join(temp_dir, 'dir*')
self.assertFalse(s_filepath.exists(path))
self.assertFalse(s_filepath.isfile(path))
self.assertFalse(s_filepath.isdir(path))
# multiple open regular files
path = os.path.join(temp_dir, 'foo*')
fds = [f for f in s_filepath.openfiles(path, mode='rb', req=False)]
self.eq(len(fds), 3)
for fd in fds:
buf = fd.read()
self.eq(len(buf), 20)
self.assertIn(buf, [f0, f1, f2])
# multiple open on dir
path = os.path.join(temp_dir, 'dir*')
def diropen(path):
[f for f in s_filepath.openfiles(path, mode='rb', req=True)]
self.assertRaises(s_exc.NoSuchPath, diropen, path)
path = os.path.join(temp_dir, 'dir*')
def diropen(path):
return [f for f in s_filepath.openfiles(path, mode='rb', req=False)]
self.eq([], diropen(path))
# multiple open on dne
path = os.path.join(temp_dir, 'dne*')
def diropen(path):
return [f for f in s_filepath.openfiles(path, mode='rb', req=True)]
self.assertRaises(s_exc.NoSuchPath, diropen, path)
ret = [ a for a in s_filepath.openfiles(None)]
self.eq([], ret)
# multiple open zip files
tzfd0 = tempfile.NamedTemporaryFile()
tzfd1 = tempfile.NamedTemporaryFile()
ttfd0 = tempfile.NamedTemporaryFile()
zfd0 = zipfile.ZipFile(tzfd0.name, 'w')
zfd0.writestr('dir0/dir1/dir2/foo0', f0)
zfd0.writestr('dir0/dir1/dir2/foo1', f1)
zfd0.writestr('dir0/dir1/dir2/foo2', f2)
zfd0.writestr('dir0/dir1/dir2/junk', 'Z'*20)
zfd0.close()
tfd0 = tarfile.TarFile(ttfd0.name, 'w')
tfd0.add(f0_path, arcname='dir0/dir1/dir2/foo0')
tfd0.add(f1_path, arcname='dir0/dir1/dir2/foo1')
tfd0.add(f2_path, arcname='dir0/dir1/dir2/foo2')
tfd0.add(f3_path, arcname='dir0/dir1/dir2/junk')
tfd0.close()
zfd1 = zipfile.ZipFile(tzfd1.name, 'w')
zfd1.writestr('dir0/dir1/dir2/bar0', f0)
zfd1.writestr('dir0/dir1/dir2/bar1', f1)
zfd1.writestr('dir0/dir1/dir2/bar2', f2)
zfd1.writestr('dir0/dir1/dir2/junk', 'Z'*20)
zfd1.write(tzfd0.name, arcname='ndir0/nested.zip')
zfd1.write(ttfd0.name, arcname='ndir0/nested.tar')
zfd1.close()
path = os.path.join(tzfd1.name, 'dir0/dir1/dir2/bar*')
count = 0
for fd in s_filepath.openfiles(path, mode='rb'):
buf = fd.read()
self.eq(len(buf), 20)
self.assertIn(buf, [f0, f1, f2])
fd.close()
count += 1
self.eq(count, 3)
tzfd0.close()
tzfd1.close()
tfd0.close()
def diropen(path):
return [f for f in s_filepath.openfiles(path, mode='rb', req=False)]
def test_filepath_glob(self):
temp_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(temp_dir, 'dir0'))
os.mkdir(os.path.join(temp_dir, 'dir1'))
os.mkdir(os.path.join(temp_dir, 'dir2'))
os.mkdir(os.path.join(temp_dir, 'fooD'))
os.makedirs(os.path.join(temp_dir, 'bazer', 'eir0', 'dir1'))
os.makedirs(os.path.join(temp_dir, 'bazer', 'eir0', 'dir2'))
f0 = b'A' * 20
f0_path = os.path.join(temp_dir, 'foo0')
with open(f0_path, 'wb') as fd:
fd.write(f0)
f1 = b'B' * 20
f1_path = os.path.join(temp_dir, 'foo1')
with open(f1_path, 'wb') as fd:
fd.write(f1)
f2 = b'C' * 20
f2_path = os.path.join(temp_dir, 'foo2')
with open(f2_path, 'wb') as fd:
fd.write(f2)
f3 = b'Z' * 20
f3_path = os.path.join(temp_dir, 'junk')
with open(f3_path, 'wb') as fd:
fd.write(f3)
# same files alpha/bravo
t1 = b'a' * 20
t_path = os.path.join(temp_dir, 'bazer', 'eir0', 'dir1', 'alpha')
with open(t_path, 'wb') as fd:
fd.write(t1)
t_path = os.path.join(temp_dir, 'bazer', 'eir0', 'dir1', 'bravo')
with open(t_path, 'wb') as fd:
fd.write(t1)
t_path = os.path.join(temp_dir, 'bazr')
with open(t_path, 'wb') as fd:
fd.write(t1)
# files that exists
path = os.path.join(temp_dir, 'foo*')
self.false(s_filepath.exists(path))
self.false(s_filepath.isfile(path))
self.false(s_filepath.isdir(path))
# dirs that exist
path = os.path.join(temp_dir, 'dir*')
self.false(s_filepath.exists(path))
self.false(s_filepath.isfile(path))
self.false(s_filepath.isdir(path))
# open a dir
fp = s_filepath.parsePath(temp_dir, 'dir0')
self.none(fp.open())
# multiple open regular files
path = os.path.join(temp_dir, 'foo*')
fd_ct = 0
f = [fd for fd in s_filepath.openfiles(path, mode='rb', req=False)]
for fd in f:
buf = fd.read()
self.eq(len(buf), 20)
self.isin(buf, [f0, f1, f2])
fd.close()
fd_ct += 1
self.eq(fd_ct, 3)
# multiple open on dir
path = os.path.join(temp_dir, 'dir*')
def diropen(path):
[f for f in s_filepath.openfiles(path, mode='rb', req=True)]
self.raises(s_exc.NoSuchPath, diropen, path)
path = os.path.join(temp_dir, 'dir*')
def diropen(path):
return [
f for f in s_filepath.openfiles(path, mode='rb', req=False)
]
self.eq([], diropen(path))
# multiple open on dne
path = os.path.join(temp_dir, 'dne*')
def diropen(path):
return [f for f in s_filepath.openfiles(path, mode='rb', req=True)]
self.raises(s_exc.NoSuchPath, diropen, path)
ret = [a for a in s_filepath.openfiles(None)]
self.eq([], ret)
# multiple open zip files
tzfd0 = open(os.path.join(temp_dir, 'baz.zip'), 'w')
tzfd1 = tempfile.NamedTemporaryFile()
ttfd0 = open(os.path.join(temp_dir, 'baz.tar'), 'w')
zfd0 = zipfile.ZipFile(tzfd0.name, 'w')
zfd0.writestr('dir0/dir1/dir2/foo0', f0)
zfd0.writestr('dir0/dir1/dir2/foo1', f1)
zfd0.writestr('dir0/dir1/dir2/foo2', f2)
zfd0.writestr('dir0/dir1/dir2/junk', 'Z' * 20)
zfd0.writestr('eir0/dir3/z1', t1)
zfd0.writestr('eir0/dir4/z2', t1)
zfd0.close()
tfd0 = tarfile.TarFile(ttfd0.name, 'w')
tfd0.add(f0_path, arcname='dir0/dir1/dir2/foo0')
tfd0.add(f1_path, arcname='dir0/dir1/dir2/foo1')
tfd0.add(f2_path, arcname='dir0/dir1/dir2/foo2')
tfd0.add(f3_path, arcname='dir0/dir1/dir2/junk')
tfd0.add(t_path, arcname='eir0/dir5/t1')
tfd0.add(t_path, arcname='eir0/dir6/t2')
tfd0.close()
zfd1 = zipfile.ZipFile(tzfd1.name, 'w')
zfd1.writestr('dir0/dir1/dir2/bar0', f0)
zfd1.writestr('dir0/dir1/dir2/bar1', f1)
zfd1.writestr('dir0/dir1/dir2/bar2', f2)
zfd1.writestr('dir0/dir1/dir2/junk', 'Z' * 20)
zfd1.write(tzfd0.name, arcname='ndir0/nested.zip')
zfd1.write(ttfd0.name, arcname='ndir0/nested.tar')
zfd1.close()
path = os.path.join(tzfd1.name, 'dir0/dir1/dir2/bar*')
count = 0
for fd in s_filepath.openfiles(path, mode='rb'):
buf = fd.read()
fd.seek(0)
self.eq(len(buf), 20)
self.eq(buf, fd.read())
self.isin(buf, [f0, f1, f2])
fd.close()
count += 1
self.eq(count, 3)
path = os.path.join(temp_dir, 'baz*', 'eir0', 'dir*', '*')
count = 0
for fd in s_filepath.openfiles(path, mode='rb'):
buf = fd.read()
self.eq(len(buf), 20)
self.eq(buf, t1)
fd.close()
count += 1
self.eq(count, 6)
tzfd0.close()
tzfd1.close()
ttfd0.close()
shutil.rmtree(temp_dir)