def test_remove_link_file(self):
src_fn = '/tmp/pykit-ut-fsutil-remove-file-normal'
force_remove(src_fn)
fsutil.write_file(src_fn, '', atomic=True)
self.assertTrue(os.path.isfile(src_fn))
link_fn = '/tmp/pykit-ut-fsutil-remove-file-link'
force_remove(link_fn)
os.link(src_fn, link_fn)
self.assertTrue(os.path.isfile(link_fn))
fsutil.remove(link_fn)
self.assertFalse(os.path.exists(link_fn))
symlink_fn = '/tmp/pykit-ut-fsutil-remove-file-symlink'
force_remove(symlink_fn)
os.symlink(src_fn, symlink_fn)
self.assertTrue(os.path.islink(symlink_fn))
fsutil.remove(symlink_fn)
self.assertFalse(os.path.exists(symlink_fn))
force_remove(src_fn)
def test_remove_dir(self):
dirname = '/tmp/pykit-ut-fsutil-remove-dir'
fsutil.makedirs(dirname)
self.assertTrue(os.path.isdir(dirname))
for is_dir, file_path in (
(False, ('normal_file', )),
(True, ('sub_dir', )),
(False, ('sub_dir', 'sub_file1')),
(False, ('sub_dir', 'sub_file2')),
(True, ('sub_empty_dir', )),
(True, ('sub_dir', 'sub_sub_dir')),
(False, ('sub_dir', 'sub_sub_dir', 'sub_sub_file')),
):
path = os.path.join(dirname, *file_path)
if is_dir:
fsutil.makedirs(path)
self.assertTrue(os.path.isdir(path))
else:
fsutil.write_file(path, '')
self.assertTrue(os.path.isfile(path))
fsutil.remove(dirname)
self.assertFalse(os.path.exists(dirname))
def test_remove_dir_with_link(self):
dirname = '/tmp/pykit-ut-fsutil-remove-dir'
fsutil.makedirs(dirname)
self.assertTrue(os.path.isdir(dirname))
normal_file = 'normal_file'
normal_path = os.path.join(dirname, normal_file)
fsutil.write_file(normal_path, '')
self.assertTrue(os.path.isfile(normal_path))
hard_link = 'hard_link'
hard_path = os.path.join(dirname, hard_link)
os.link(normal_path, hard_path)
self.assertTrue(os.path.isfile(hard_path))
symbolic_link = 'symbolic_link'
symbolic_path = os.path.join(dirname, symbolic_link)
os.symlink(hard_path, symbolic_path)
self.assertTrue(os.path.islink(symbolic_path))
fsutil.remove(dirname)
self.assertFalse(os.path.exists(dirname))
def test_data_chucked(self):
expected = ['a' * 32]
new_data = ['b' * 32]
chucked = ['c' * 31]
cat_handle = fsutil.Cat(self.fn, strip=True)
rst = []
append_lines(self.fn, expected)
for l in cat_handle.iterate(timeout=0):
rst.append(l)
self.assertEqual(expected, rst)
# file was refreshed
os.rename(self.fn, self.fn + '_old')
append_lines(self.fn, new_data)
fsutil.remove(self.fn + '_old')
for l in cat_handle.iterate(timeout=0):
rst.append(l)
self.assertEqual(expected + new_data, rst)
# file was chucked
fsutil.write_file(self.fn, chucked[0])
for l in cat_handle.iterate(timeout=0):
rst.append(l)
dd(rst)
self.assertEqual(expected + new_data + chucked, rst)
def test_get_sub_dirs(self):
fsutil.makedirs('test_dir/sub_dir1')
fsutil.makedirs('test_dir/sub_dir2')
fsutil.write_file('test_dir/test_file', 'foo')
sub_dirs = fsutil.get_sub_dirs('test_dir')
self.assertListEqual(['sub_dir1', 'sub_dir2'], sub_dirs)
fsutil.remove('test_dir')
def test_remove_normal_file(self):
fn = '/tmp/pykit-ut-fsutil-remove-file-normal'
force_remove(fn)
fsutil.write_file(fn, '', atomic=True)
self.assertTrue(os.path.isfile(fn))
fsutil.remove(fn)
self.assertFalse(os.path.exists(fn))
def test_list_fns(self):
fsutil.makedirs('test_dir/foo_dir')
fsutil.write_file('test_dir/foo1', 'foo1')
fsutil.write_file('test_dir/foo2', 'foo2')
fsutil.write_file('test_dir/foo21', 'foo21')
fsutil.write_file('test_dir/foo_dir/foo', 'foo')
fsutil.write_file('test_dir/foo_dir/bar', 'bar')
self.assertEqual(['foo1', 'foo2', 'foo21'],
fsutil.list_fns('test_dir'))
self.assertEqual(['foo2'], fsutil.list_fns('test_dir', pattern='2$'))
self.assertEqual(['bar', 'foo'], fsutil.list_fns('test_dir/foo_dir'))
self.assertEqual(['bar'],
fsutil.list_fns('test_dir/foo_dir', pattern='^b'))
fsutil.remove('test_dir')
def test_offset_record_when_destory(self):
expected = [x * 32 for x in 'qwertyuiop']
append_lines(self.fn, expected)
cat_handle = fsutil.Cat(self.fn, strip=True)
rst = []
for val in cat_handle.iterate(timeout=0):
rst.append(val)
self.assertEqual(expected, rst)
# stat file was removed
fsutil.remove(cat_handle.stat_path())
for val in cat_handle.iterate(timeout=0):
rst.append(val)
self.assertEqual(expected * 2, rst)
# stat file was damaged
fsutil.write_file(cat_handle.stat_path(), '{]')
for val in cat_handle.iterate(timeout=0):
rst.append(val)
self.assertEqual(expected * 3, rst)
def test_remove_error(self):
dirname = '/tmp/pykit-ut-fsutil-remove-on-error'
if os.path.isdir(dirname):
fsutil.remove(dirname)
# OSError
self.assertRaises(os.error, fsutil.remove, dirname, False)
# ignore errors
fsutil.remove(dirname, ignore_errors=True)
def assert_error(exp_func):
def onerror(func, path, exc_info):
self.assertEqual(func, exp_func)
return onerror
# on error
fsutil.remove(dirname, onerror=assert_error(os.remove))
class TestMultipart(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(TestMultipart, self).__init__(*args, **kwargs)
self.test_multipart = None
def setUp(self):
self.test_multipart = httpmultipart.Multipart()
def test_headers(self):
case = [[{
'Content-Length': 998,
'Content-Type': 'aaplication/octet-stream'
}, {
'Content-Length':
998,
'Content-Type':
'multipart/form-data; ' +
'boundary={b}'.format(b=self.test_multipart.boundary)
}],
[{
'Content-Length': 1200
}, {
'Content-Length':
1200,
'Content-Type':
'multipart/form-data; ' +
'boundary={b}'.format(b=self.test_multipart.boundary)
}],
[{
'Content-Type': 'application/octet-stream'
}, {
'Content-Length':
1335,
'Content-Type':
'multipart/form-data; ' +
'boundary={b}'.format(b=self.test_multipart.boundary)
}],
[
None, {
'Content-Length':
1335,
'Content-Type':
'multipart/form-data; ' +
'boundary={b}'.format(b=self.test_multipart.boundary)
}
]]
str1 = '''
使命：Better Internet ，Better life
愿景：成为全球受人尊敬的科技公司；最具创新力；最佳雇主
未来白山的特质：渴求变革；让创新超越客户想象；全球化；真诚、并始终如一
信条：以用户为中心，其他一切水到渠成；专心将一件事做到极致；越快越好
'''
str2 = '''
12343564343rfe
fdf4erguu38788894hf
12rfhfvh8876w91908777yfj
'''
str3 = '''
838839938238838388383838
dddjjdkkksijdidhdhhhhddd
djdjdfdf4erguu38788894hf
12rfhfvh8876w91908777yfj
'''
fsutil.write_file('/tmp/a.txt', str1)
fsutil.write_file('/tmp/b.txt', str2)
def make_file_reader():
with open('/tmp/b.txt') as f:
while True:
buf = f.read(1024 * 1024)
if buf == '':
break
yield buf
def make_str_reader():
yield str3
str_reader = make_str_reader()
str_size = len(str3)
file_reader = make_file_reader()
file_size = os.path.getsize('/tmp/b.txt')
fields = [
{
'name': 'metadata1',
'value': 'lvting',
'headers': {
'Date': 'Dec, 20 Dec 2018 15:00:00 GMT'
}
},
{
'name': 'metadata2',
'value': [
open('/tmp/a.txt'),
os.path.getsize('/tmp/a.txt'),
],
'headers': {
'Content-Type': 'application/octet-stream'
}
},
{
'name': 'metadata3',
'value': [file_reader, file_size, 'b.txt'],
},
{
'name': 'metadata4',
'value': [str_reader, str_size],
},
{
'name': 'metadata5',
'value': ['234ffhhif3323jjfjf3']
},
]
for h in case:
self.assertEqual(h[1],
self.test_multipart.make_headers(fields, h[0]))
fsutil.remove('/tmp/a.txt')
fsutil.remove('/tmp/b.txt')
def test_headers(self):
case = [
[
{
'Content-Length': 998,
'Content-Type': 'aaplication/octet-stream'
},
{
'Content-Length': 998,
'Content-Type': 'multipart/form-data; ' +
'boundary={b}'.format(b=self.test_multipart.boundary)
}
],
[
{
'Content-Length': 1200
},
{
'Content-Length': 1200,
'Content-Type': 'multipart/form-data; ' +
'boundary={b}'.format(b=self.test_multipart.boundary)
}
],
[
{
'Content-Type': 'application/octet-stream'
},
{
'Content-Length': 1335,
'Content-Type': 'multipart/form-data; ' +
'boundary={b}'.format(b=self.test_multipart.boundary)
}
],
[
None,
{
'Content-Length': 1335,
'Content-Type': 'multipart/form-data; ' +
'boundary={b}'.format(b=self.test_multipart.boundary)
}
]
]
str1 = '''
使命：Better Internet ，Better life
愿景：成为全球受人尊敬的科技公司；最具创新力；最佳雇主
未来白山的特质：渴求变革；让创新超越客户想象；全球化；真诚、并始终如一
信条：以用户为中心，其他一切水到渠成；专心将一件事做到极致；越快越好
'''
str2 = '''
12343564343rfe
fdf4erguu38788894hf
12rfhfvh8876w91908777yfj
'''
str3 = '''
838839938238838388383838
dddjjdkkksijdidhdhhhhddd
djdjdfdf4erguu38788894hf
12rfhfvh8876w91908777yfj
'''
fsutil.write_file(
'/tmp/a.txt', str1
)
fsutil.write_file(
'/tmp/b.txt', str2
)
def make_file_reader():
with open('/tmp/b.txt') as f:
while True:
buf = f.read(1024 * 1024)
if buf == '':
break
yield buf
def make_str_reader():
yield str3
str_reader = make_str_reader()
str_size = len(str3)
file_reader = make_file_reader()
file_size = os.path.getsize('/tmp/b.txt')
fields = [
{
'name': 'metadata1',
'value': 'lvting',
'headers': {'Date': 'Dec, 20 Dec 2018 15:00:00 GMT'}
},
{
'name': 'metadata2',
'value': [
open('/tmp/a.txt'),
os.path.getsize('/tmp/a.txt'),
],
'headers': {'Content-Type': 'application/octet-stream'}
},
{
'name': 'metadata3',
'value': [file_reader, file_size, 'b.txt'],
},
{
'name': 'metadata4',
'value': [str_reader, str_size],
},
{
'name': 'metadata5',
'value': ['234ffhhif3323jjfjf3']
},
]
for h in case:
self.assertEqual(
h[1],
self.test_multipart.make_headers(fields, h[0])
)
fsutil.remove('/tmp/a.txt')
fsutil.remove('/tmp/b.txt')
def test_body(self):
str1 = '''
使命：Better Internet ，Better life
愿景：成为全球受人尊敬的科技公司；最具创新力；最佳雇主
未来白山的特质：渴求变革；让创新超越客户想象；全球化；真诚、并始终如一
信条：以用户为中心，其他一切水到渠成；专心将一件事做到极致；越快越好
'''
str2 = '''
12343564343rfe
fdf4erguu38788894hf
12rfhfvh8876w91908777yfj
'''
str3 = '''
838839938238838388383838
dddjjdkkksijdidhdhhhhddd
djdjdfdf4erguu38788894hf
12rfhfvh8876w91908777yfj
'''
fsutil.write_file(
'/tmp/a.txt', str1
)
fsutil.write_file(
'/tmp/b.txt', str2
)
def make_file_reader():
with open('/tmp/b.txt') as f:
while True:
buf = f.read(1024 * 1024)
if buf == '':
break
yield buf
def make_str_reader():
yield str3
str_reader = make_str_reader()
str_size = len(str3)
file_reader = make_file_reader()
file_size = os.path.getsize('/tmp/b.txt')
case = [
[
[
{
'name': 'metadata1',
'value': 'lvting',
'headers': {'Date': 'Dec, 20 Dec 2018 15:00:00 GMT'}
}
],
[
'--{b}'.format(b=self.test_multipart.boundary),
'Content-Disposition: form-data; name=metadata1',
'Date: Dec, 20 Dec 2018 15:00:00 GMT',
'',
'lvting',
'--{b}--'.format(b=self.test_multipart.boundary),
]
],
[
[
{
'name': 'metadata2',
'value': [
open('/tmp/a.txt'),
os.path.getsize('/tmp/a.txt')
],
}
],
[
'--{b}'.format(b=self.test_multipart.boundary),
'Content-Disposition: form-data; name=metadata2',
'',
str1,
'--{b}--'.format(b=self.test_multipart.boundary),
]
],
[
[
{
'name': 'metadata3',
'value': [file_reader, file_size, 'b.txt'],
}
],
[
'--{b}'.format(b=self.test_multipart.boundary),
'Content-Disposition: form-data; name=metadata3; '
+ 'filename=b.txt',
'Content-Type: text/plain',
'',
str2,
'--{b}--'.format(b=self.test_multipart.boundary),
]
],
[
[
{
'name': 'metadata4',
'value': [str_reader, str_size],
}
],
[
'--{b}'.format(b=self.test_multipart.boundary),
'Content-Disposition: form-data; name=metadata4',
'',
str3,
'--{b}--'.format(b=self.test_multipart.boundary),
]
],
[
[
{
'name': 'metadata5',
'value': ['234ffhhif3323jjfjf3']
}
],
[
'--{b}'.format(b=self.test_multipart.boundary),
'Content-Disposition: form-data; name=metadata5',
'',
'234ffhhif3323jjfjf3',
'--{b}--'.format(b=self.test_multipart.boundary),
]
],
]
for c in case:
body = self.test_multipart.make_body_reader(c[0])
data = []
for x in body:
data.append(x)
self.assertEqual('\r\n'.join(c[1]), ''.join(data))
fsutil.remove('/tmp/a.txt')
fsutil.remove('/tmp/b.txt')
def test_blkio_weight(self):
if ututil.has_env('TRAVIS=true'):
return
manager = multiprocessing.Manager()
result_dict = manager.dict()
p1 = multiprocessing.Process(target=self.worker,
args=(1, 10, result_dict))
p1.daemon = True
p1.start()
p2 = multiprocessing.Process(target=self.worker,
args=(2, 10, result_dict))
p2.daemon = True
p2.start()
p3 = multiprocessing.Process(target=self.worker,
args=(3, 10, result_dict))
p3.daemon = True
p3.start()
p4 = multiprocessing.Process(target=self.worker,
args=(4, 10, result_dict))
p4.daemon = True
p4.start()
arch_conf = {
'blkio': {
'sub_cgroup': {
'test_cgroup_a': {
'conf': {
'weight': int(500 * 0.95),
},
'sub_cgroup': {
'test_cgroup_a_sub1': {
'conf': {
'weight': 500,
'pids': [p1.pid],
},
},
'test_cgroup_a_sub2': {
'conf': {
'weight': 500,
'pids': [p2.pid],
},
},
},
},
'test_cgroup_b': {
'conf': {
'weight': int(500 * 0.05),
},
'sub_cgroup': {
'test_cgroup_b_sub1': {
'conf': {
'weight': 500,
'pids': [p3.pid],
},
},
'test_cgroup_b_sub2': {
'conf': {
'weight': 500,
'pids': [p4.pid],
},
},
},
},
},
},
}
context = {
'cgroup_dir': '/sys/fs/cgroup',
'arch_conf': {
'value': arch_conf
},
}
cgroup_manager.build_all_subsystem_cgroup_arch(context)
cgroup_manager.set_cgroup(context)
p1.join()
p2.join()
p3.join()
p4.join()
for cgrou_name in arch_conf['blkio']['sub_cgroup'].keys():
cgroup_util.remove_cgroup(
os.path.join(context['cgroup_dir'], 'blkio'),
os.path.join(context['cgroup_dir'], 'blkio', cgrou_name))
for i in range(1, 5):
fsutil.remove(os.path.join(base_dir, 'test_file_%d' % i))
dd(result_dict)
self.assertGreater(result_dict[1] + result_dict[2],
result_dict[3] + result_dict[4])
def tearDownClass(cls):
cmd = "umount '{_dir}'".format(_dir=cls.testArc)
proc.command(cmd, shell=True)
fsutil.remove(cls.testArc)
def tearDown(self):
fsutil.remove(self.testSrc)
for d in fsutil.get_sub_dirs(self.testArc):
fsutil.remove(os.path.join(self.testArc, d))
def clear_progress():
fsutil.remove(cnf['PROGRESS_FILE'])
def test_body(self):
str1 = '''
使命：Better Internet ，Better life
愿景：成为全球受人尊敬的科技公司；最具创新力；最佳雇主
未来白山的特质：渴求变革；让创新超越客户想象；全球化；真诚、并始终如一
信条：以用户为中心，其他一切水到渠成；专心将一件事做到极致；越快越好
'''
str2 = '''
12343564343rfe
fdf4erguu38788894hf
12rfhfvh8876w91908777yfj
'''
str3 = '''
838839938238838388383838
dddjjdkkksijdidhdhhhhddd
djdjdfdf4erguu38788894hf
12rfhfvh8876w91908777yfj
'''
fsutil.write_file('/tmp/a.txt', str1)
fsutil.write_file('/tmp/b.txt', str2)
def make_file_reader():
with open('/tmp/b.txt') as f:
while True:
buf = f.read(1024 * 1024)
if buf == '':
break
yield buf
def make_str_reader():
yield str3
str_reader = make_str_reader()
str_size = len(str3)
file_reader = make_file_reader()
file_size = os.path.getsize('/tmp/b.txt')
case = [
[[{
'name': 'metadata1',
'value': 'lvting',
'headers': {
'Date': 'Dec, 20 Dec 2018 15:00:00 GMT'
}
}],
[
'--{b}'.format(b=self.test_multipart.boundary),
'Content-Disposition: form-data; name=metadata1',
'Date: Dec, 20 Dec 2018 15:00:00 GMT',
'',
'lvting',
'--{b}--'.format(b=self.test_multipart.boundary),
]],
[[{
'name': 'metadata2',
'value': [open('/tmp/a.txt'),
os.path.getsize('/tmp/a.txt')],
}],
[
'--{b}'.format(b=self.test_multipart.boundary),
'Content-Disposition: form-data; name=metadata2',
'',
str1,
'--{b}--'.format(b=self.test_multipart.boundary),
]],
[[{
'name': 'metadata3',
'value': [file_reader, file_size, 'b.txt'],
}],
[
'--{b}'.format(b=self.test_multipart.boundary),
'Content-Disposition: form-data; name=metadata3; ' +
'filename=b.txt',
'Content-Type: text/plain',
'',
str2,
'--{b}--'.format(b=self.test_multipart.boundary),
]],
[[{
'name': 'metadata4',
'value': [str_reader, str_size],
}],
[
'--{b}'.format(b=self.test_multipart.boundary),
'Content-Disposition: form-data; name=metadata4',
'',
str3,
'--{b}--'.format(b=self.test_multipart.boundary),
]],
[[{
'name': 'metadata5',
'value': ['234ffhhif3323jjfjf3']
}],
[
'--{b}'.format(b=self.test_multipart.boundary),
'Content-Disposition: form-data; name=metadata5',
'',
'234ffhhif3323jjfjf3',
'--{b}--'.format(b=self.test_multipart.boundary),
]],
]
for c in case:
body = self.test_multipart.make_body_reader(c[0])
data = []
for x in body:
data.append(x)
self.assertEqual('\r\n'.join(c[1]), ''.join(data))
fsutil.remove('/tmp/a.txt')
fsutil.remove('/tmp/b.txt')
