def test_remove_link_file(self): src_fn = '/tmp/pykit-ut-fsutil-remove-file-normal' force_remove(src_fn) fsutil.write_file(src_fn, '', atomic=True) self.assertTrue(os.path.isfile(src_fn)) link_fn = '/tmp/pykit-ut-fsutil-remove-file-link' force_remove(link_fn) os.link(src_fn, link_fn) self.assertTrue(os.path.isfile(link_fn)) fsutil.remove(link_fn) self.assertFalse(os.path.exists(link_fn)) symlink_fn = '/tmp/pykit-ut-fsutil-remove-file-symlink' force_remove(symlink_fn) os.symlink(src_fn, symlink_fn) self.assertTrue(os.path.islink(symlink_fn)) fsutil.remove(symlink_fn) self.assertFalse(os.path.exists(symlink_fn)) force_remove(src_fn)
def test_remove_dir(self): dirname = '/tmp/pykit-ut-fsutil-remove-dir' fsutil.makedirs(dirname) self.assertTrue(os.path.isdir(dirname)) for is_dir, file_path in ( (False, ('normal_file', )), (True, ('sub_dir', )), (False, ('sub_dir', 'sub_file1')), (False, ('sub_dir', 'sub_file2')), (True, ('sub_empty_dir', )), (True, ('sub_dir', 'sub_sub_dir')), (False, ('sub_dir', 'sub_sub_dir', 'sub_sub_file')), ): path = os.path.join(dirname, *file_path) if is_dir: fsutil.makedirs(path) self.assertTrue(os.path.isdir(path)) else: fsutil.write_file(path, '') self.assertTrue(os.path.isfile(path)) fsutil.remove(dirname) self.assertFalse(os.path.exists(dirname))
def test_remove_dir_with_link(self): dirname = '/tmp/pykit-ut-fsutil-remove-dir' fsutil.makedirs(dirname) self.assertTrue(os.path.isdir(dirname)) normal_file = 'normal_file' normal_path = os.path.join(dirname, normal_file) fsutil.write_file(normal_path, '') self.assertTrue(os.path.isfile(normal_path)) hard_link = 'hard_link' hard_path = os.path.join(dirname, hard_link) os.link(normal_path, hard_path) self.assertTrue(os.path.isfile(hard_path)) symbolic_link = 'symbolic_link' symbolic_path = os.path.join(dirname, symbolic_link) os.symlink(hard_path, symbolic_path) self.assertTrue(os.path.islink(symbolic_path)) fsutil.remove(dirname) self.assertFalse(os.path.exists(dirname))
def test_data_chucked(self): expected = ['a' * 32] new_data = ['b' * 32] chucked = ['c' * 31] cat_handle = fsutil.Cat(self.fn, strip=True) rst = [] append_lines(self.fn, expected) for l in cat_handle.iterate(timeout=0): rst.append(l) self.assertEqual(expected, rst) # file was refreshed os.rename(self.fn, self.fn + '_old') append_lines(self.fn, new_data) fsutil.remove(self.fn + '_old') for l in cat_handle.iterate(timeout=0): rst.append(l) self.assertEqual(expected + new_data, rst) # file was chucked fsutil.write_file(self.fn, chucked[0]) for l in cat_handle.iterate(timeout=0): rst.append(l) dd(rst) self.assertEqual(expected + new_data + chucked, rst)
def test_get_sub_dirs(self): fsutil.makedirs('test_dir/sub_dir1') fsutil.makedirs('test_dir/sub_dir2') fsutil.write_file('test_dir/test_file', 'foo') sub_dirs = fsutil.get_sub_dirs('test_dir') self.assertListEqual(['sub_dir1', 'sub_dir2'], sub_dirs) fsutil.remove('test_dir')
def test_remove_normal_file(self): fn = '/tmp/pykit-ut-fsutil-remove-file-normal' force_remove(fn) fsutil.write_file(fn, '', atomic=True) self.assertTrue(os.path.isfile(fn)) fsutil.remove(fn) self.assertFalse(os.path.exists(fn))
def test_list_fns(self): fsutil.makedirs('test_dir/foo_dir') fsutil.write_file('test_dir/foo1', 'foo1') fsutil.write_file('test_dir/foo2', 'foo2') fsutil.write_file('test_dir/foo21', 'foo21') fsutil.write_file('test_dir/foo_dir/foo', 'foo') fsutil.write_file('test_dir/foo_dir/bar', 'bar') self.assertEqual(['foo1', 'foo2', 'foo21'], fsutil.list_fns('test_dir')) self.assertEqual(['foo2'], fsutil.list_fns('test_dir', pattern='2$')) self.assertEqual(['bar', 'foo'], fsutil.list_fns('test_dir/foo_dir')) self.assertEqual(['bar'], fsutil.list_fns('test_dir/foo_dir', pattern='^b')) fsutil.remove('test_dir')
def test_offset_record_when_destory(self): expected = [x * 32 for x in 'qwertyuiop'] append_lines(self.fn, expected) cat_handle = fsutil.Cat(self.fn, strip=True) rst = [] for val in cat_handle.iterate(timeout=0): rst.append(val) self.assertEqual(expected, rst) # stat file was removed fsutil.remove(cat_handle.stat_path()) for val in cat_handle.iterate(timeout=0): rst.append(val) self.assertEqual(expected * 2, rst) # stat file was damaged fsutil.write_file(cat_handle.stat_path(), '{]') for val in cat_handle.iterate(timeout=0): rst.append(val) self.assertEqual(expected * 3, rst)
def test_remove_error(self): dirname = '/tmp/pykit-ut-fsutil-remove-on-error' if os.path.isdir(dirname): fsutil.remove(dirname) # OSError self.assertRaises(os.error, fsutil.remove, dirname, False) # ignore errors fsutil.remove(dirname, ignore_errors=True) def assert_error(exp_func): def onerror(func, path, exc_info): self.assertEqual(func, exp_func) return onerror # on error fsutil.remove(dirname, onerror=assert_error(os.remove))
class TestMultipart(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestMultipart, self).__init__(*args, **kwargs) self.test_multipart = None def setUp(self): self.test_multipart = httpmultipart.Multipart() def test_headers(self): case = [[{ 'Content-Length': 998, 'Content-Type': 'aaplication/octet-stream' }, { 'Content-Length': 998, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) }], [{ 'Content-Length': 1200 }, { 'Content-Length': 1200, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) }], [{ 'Content-Type': 'application/octet-stream' }, { 'Content-Length': 1335, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) }], [ None, { 'Content-Length': 1335, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) } ]] str1 = ''' 使命:Better Internet ,Better life 愿景:成为全球受人尊敬的科技公司;最具创新力;最佳雇主 未来白山的特质:渴求变革;让创新超越客户想象;全球化;真诚、并始终如一 信条:以用户为中心,其他一切水到渠成;专心将一件事做到极致;越快越好 ''' str2 = ''' 12343564343rfe fdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' str3 = ''' 838839938238838388383838 dddjjdkkksijdidhdhhhhddd djdjdfdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' fsutil.write_file('/tmp/a.txt', str1) fsutil.write_file('/tmp/b.txt', str2) def make_file_reader(): with open('/tmp/b.txt') as f: while True: buf = f.read(1024 * 1024) if buf == '': break yield buf def make_str_reader(): yield str3 str_reader = make_str_reader() str_size = len(str3) file_reader = make_file_reader() file_size = os.path.getsize('/tmp/b.txt') fields = [ { 'name': 'metadata1', 'value': 'lvting', 'headers': { 'Date': 'Dec, 20 Dec 2018 15:00:00 GMT' } }, { 'name': 'metadata2', 'value': [ open('/tmp/a.txt'), os.path.getsize('/tmp/a.txt'), ], 'headers': { 'Content-Type': 'application/octet-stream' } }, { 'name': 'metadata3', 'value': [file_reader, file_size, 'b.txt'], }, { 'name': 'metadata4', 'value': [str_reader, str_size], }, { 'name': 'metadata5', 'value': ['234ffhhif3323jjfjf3'] }, ] for h in case: self.assertEqual(h[1], self.test_multipart.make_headers(fields, h[0])) fsutil.remove('/tmp/a.txt') fsutil.remove('/tmp/b.txt')
def test_headers(self): case = [ [ { 'Content-Length': 998, 'Content-Type': 'aaplication/octet-stream' }, { 'Content-Length': 998, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) } ], [ { 'Content-Length': 1200 }, { 'Content-Length': 1200, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) } ], [ { 'Content-Type': 'application/octet-stream' }, { 'Content-Length': 1335, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) } ], [ None, { 'Content-Length': 1335, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) } ] ] str1 = ''' 使命:Better Internet ,Better life 愿景:成为全球受人尊敬的科技公司;最具创新力;最佳雇主 未来白山的特质:渴求变革;让创新超越客户想象;全球化;真诚、并始终如一 信条:以用户为中心,其他一切水到渠成;专心将一件事做到极致;越快越好 ''' str2 = ''' 12343564343rfe fdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' str3 = ''' 838839938238838388383838 dddjjdkkksijdidhdhhhhddd djdjdfdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' fsutil.write_file( '/tmp/a.txt', str1 ) fsutil.write_file( '/tmp/b.txt', str2 ) def make_file_reader(): with open('/tmp/b.txt') as f: while True: buf = f.read(1024 * 1024) if buf == '': break yield buf def make_str_reader(): yield str3 str_reader = make_str_reader() str_size = len(str3) file_reader = make_file_reader() file_size = os.path.getsize('/tmp/b.txt') fields = [ { 'name': 'metadata1', 'value': 'lvting', 'headers': {'Date': 'Dec, 20 Dec 2018 15:00:00 GMT'} }, { 'name': 'metadata2', 'value': [ open('/tmp/a.txt'), os.path.getsize('/tmp/a.txt'), ], 'headers': {'Content-Type': 'application/octet-stream'} }, { 'name': 'metadata3', 'value': [file_reader, file_size, 'b.txt'], }, { 'name': 'metadata4', 'value': [str_reader, str_size], }, { 'name': 'metadata5', 'value': ['234ffhhif3323jjfjf3'] }, ] for h in case: self.assertEqual( h[1], self.test_multipart.make_headers(fields, h[0]) ) fsutil.remove('/tmp/a.txt') fsutil.remove('/tmp/b.txt')
def test_body(self): str1 = ''' 使命:Better Internet ,Better life 愿景:成为全球受人尊敬的科技公司;最具创新力;最佳雇主 未来白山的特质:渴求变革;让创新超越客户想象;全球化;真诚、并始终如一 信条:以用户为中心,其他一切水到渠成;专心将一件事做到极致;越快越好 ''' str2 = ''' 12343564343rfe fdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' str3 = ''' 838839938238838388383838 dddjjdkkksijdidhdhhhhddd djdjdfdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' fsutil.write_file( '/tmp/a.txt', str1 ) fsutil.write_file( '/tmp/b.txt', str2 ) def make_file_reader(): with open('/tmp/b.txt') as f: while True: buf = f.read(1024 * 1024) if buf == '': break yield buf def make_str_reader(): yield str3 str_reader = make_str_reader() str_size = len(str3) file_reader = make_file_reader() file_size = os.path.getsize('/tmp/b.txt') case = [ [ [ { 'name': 'metadata1', 'value': 'lvting', 'headers': {'Date': 'Dec, 20 Dec 2018 15:00:00 GMT'} } ], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata1', 'Date: Dec, 20 Dec 2018 15:00:00 GMT', '', 'lvting', '--{b}--'.format(b=self.test_multipart.boundary), ] ], [ [ { 'name': 'metadata2', 'value': [ open('/tmp/a.txt'), os.path.getsize('/tmp/a.txt') ], } ], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata2', '', str1, '--{b}--'.format(b=self.test_multipart.boundary), ] ], [ [ { 'name': 'metadata3', 'value': [file_reader, file_size, 'b.txt'], } ], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata3; ' + 'filename=b.txt', 'Content-Type: text/plain', '', str2, '--{b}--'.format(b=self.test_multipart.boundary), ] ], [ [ { 'name': 'metadata4', 'value': [str_reader, str_size], } ], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata4', '', str3, '--{b}--'.format(b=self.test_multipart.boundary), ] ], [ [ { 'name': 'metadata5', 'value': ['234ffhhif3323jjfjf3'] } ], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata5', '', '234ffhhif3323jjfjf3', '--{b}--'.format(b=self.test_multipart.boundary), ] ], ] for c in case: body = self.test_multipart.make_body_reader(c[0]) data = [] for x in body: data.append(x) self.assertEqual('\r\n'.join(c[1]), ''.join(data)) fsutil.remove('/tmp/a.txt') fsutil.remove('/tmp/b.txt')
def test_blkio_weight(self): if ututil.has_env('TRAVIS=true'): return manager = multiprocessing.Manager() result_dict = manager.dict() p1 = multiprocessing.Process(target=self.worker, args=(1, 10, result_dict)) p1.daemon = True p1.start() p2 = multiprocessing.Process(target=self.worker, args=(2, 10, result_dict)) p2.daemon = True p2.start() p3 = multiprocessing.Process(target=self.worker, args=(3, 10, result_dict)) p3.daemon = True p3.start() p4 = multiprocessing.Process(target=self.worker, args=(4, 10, result_dict)) p4.daemon = True p4.start() arch_conf = { 'blkio': { 'sub_cgroup': { 'test_cgroup_a': { 'conf': { 'weight': int(500 * 0.95), }, 'sub_cgroup': { 'test_cgroup_a_sub1': { 'conf': { 'weight': 500, 'pids': [p1.pid], }, }, 'test_cgroup_a_sub2': { 'conf': { 'weight': 500, 'pids': [p2.pid], }, }, }, }, 'test_cgroup_b': { 'conf': { 'weight': int(500 * 0.05), }, 'sub_cgroup': { 'test_cgroup_b_sub1': { 'conf': { 'weight': 500, 'pids': [p3.pid], }, }, 'test_cgroup_b_sub2': { 'conf': { 'weight': 500, 'pids': [p4.pid], }, }, }, }, }, }, } context = { 'cgroup_dir': '/sys/fs/cgroup', 'arch_conf': { 'value': arch_conf }, } cgroup_manager.build_all_subsystem_cgroup_arch(context) cgroup_manager.set_cgroup(context) p1.join() p2.join() p3.join() p4.join() for cgrou_name in arch_conf['blkio']['sub_cgroup'].keys(): cgroup_util.remove_cgroup( os.path.join(context['cgroup_dir'], 'blkio'), os.path.join(context['cgroup_dir'], 'blkio', cgrou_name)) for i in range(1, 5): fsutil.remove(os.path.join(base_dir, 'test_file_%d' % i)) dd(result_dict) self.assertGreater(result_dict[1] + result_dict[2], result_dict[3] + result_dict[4])
def tearDownClass(cls): cmd = "umount '{_dir}'".format(_dir=cls.testArc) proc.command(cmd, shell=True) fsutil.remove(cls.testArc)
def tearDown(self): fsutil.remove(self.testSrc) for d in fsutil.get_sub_dirs(self.testArc): fsutil.remove(os.path.join(self.testArc, d))
def clear_progress(): fsutil.remove(cnf['PROGRESS_FILE'])
def test_body(self): str1 = ''' 使命:Better Internet ,Better life 愿景:成为全球受人尊敬的科技公司;最具创新力;最佳雇主 未来白山的特质:渴求变革;让创新超越客户想象;全球化;真诚、并始终如一 信条:以用户为中心,其他一切水到渠成;专心将一件事做到极致;越快越好 ''' str2 = ''' 12343564343rfe fdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' str3 = ''' 838839938238838388383838 dddjjdkkksijdidhdhhhhddd djdjdfdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' fsutil.write_file('/tmp/a.txt', str1) fsutil.write_file('/tmp/b.txt', str2) def make_file_reader(): with open('/tmp/b.txt') as f: while True: buf = f.read(1024 * 1024) if buf == '': break yield buf def make_str_reader(): yield str3 str_reader = make_str_reader() str_size = len(str3) file_reader = make_file_reader() file_size = os.path.getsize('/tmp/b.txt') case = [ [[{ 'name': 'metadata1', 'value': 'lvting', 'headers': { 'Date': 'Dec, 20 Dec 2018 15:00:00 GMT' } }], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata1', 'Date: Dec, 20 Dec 2018 15:00:00 GMT', '', 'lvting', '--{b}--'.format(b=self.test_multipart.boundary), ]], [[{ 'name': 'metadata2', 'value': [open('/tmp/a.txt'), os.path.getsize('/tmp/a.txt')], }], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata2', '', str1, '--{b}--'.format(b=self.test_multipart.boundary), ]], [[{ 'name': 'metadata3', 'value': [file_reader, file_size, 'b.txt'], }], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata3; ' + 'filename=b.txt', 'Content-Type: text/plain', '', str2, '--{b}--'.format(b=self.test_multipart.boundary), ]], [[{ 'name': 'metadata4', 'value': [str_reader, str_size], }], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata4', '', str3, '--{b}--'.format(b=self.test_multipart.boundary), ]], [[{ 'name': 'metadata5', 'value': ['234ffhhif3323jjfjf3'] }], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata5', '', '234ffhhif3323jjfjf3', '--{b}--'.format(b=self.test_multipart.boundary), ]], ] for c in case: body = self.test_multipart.make_body_reader(c[0]) data = [] for x in body: data.append(x) self.assertEqual('\r\n'.join(c[1]), ''.join(data)) fsutil.remove('/tmp/a.txt') fsutil.remove('/tmp/b.txt')