def test_remove_link_file(self): src_fn = '/tmp/pykit-ut-fsutil-remove-file-normal' force_remove(src_fn) fsutil.write_file(src_fn, '', atomic=True) self.assertTrue(os.path.isfile(src_fn)) link_fn = '/tmp/pykit-ut-fsutil-remove-file-link' force_remove(link_fn) os.link(src_fn, link_fn) self.assertTrue(os.path.isfile(link_fn)) fsutil.remove(link_fn) self.assertFalse(os.path.exists(link_fn)) symlink_fn = '/tmp/pykit-ut-fsutil-remove-file-symlink' force_remove(symlink_fn) os.symlink(src_fn, symlink_fn) self.assertTrue(os.path.islink(symlink_fn)) fsutil.remove(symlink_fn) self.assertFalse(os.path.exists(symlink_fn)) force_remove(src_fn)
def test_data_chucked(self): expected = ['a' * 32] new_data = ['b' * 32] chucked = ['c' * 31] cat_handle = fsutil.Cat(self.fn, strip=True) rst = [] append_lines(self.fn, expected) for l in cat_handle.iterate(timeout=0): rst.append(l) self.assertEqual(expected, rst) # file was refreshed os.rename(self.fn, self.fn + '_old') append_lines(self.fn, new_data) fsutil.remove(self.fn + '_old') for l in cat_handle.iterate(timeout=0): rst.append(l) self.assertEqual(expected + new_data, rst) # file was chucked fsutil.write_file(self.fn, chucked[0]) for l in cat_handle.iterate(timeout=0): rst.append(l) dd(rst) self.assertEqual(expected + new_data + chucked, rst)
def test_remove_dir_with_link(self): dirname = '/tmp/pykit-ut-fsutil-remove-dir' fsutil.makedirs(dirname) self.assertTrue(os.path.isdir(dirname)) normal_file = 'normal_file' normal_path = os.path.join(dirname, normal_file) fsutil.write_file(normal_path, '') self.assertTrue(os.path.isfile(normal_path)) hard_link = 'hard_link' hard_path = os.path.join(dirname, hard_link) os.link(normal_path, hard_path) self.assertTrue(os.path.isfile(hard_path)) symbolic_link = 'symbolic_link' symbolic_path = os.path.join(dirname, symbolic_link) os.symlink(hard_path, symbolic_path) self.assertTrue(os.path.islink(symbolic_path)) fsutil.remove(dirname) self.assertFalse(os.path.exists(dirname))
def test_remove_dir(self): dirname = '/tmp/pykit-ut-fsutil-remove-dir' fsutil.makedirs(dirname) self.assertTrue(os.path.isdir(dirname)) for is_dir, file_path in ( (False, ('normal_file', )), (True, ('sub_dir', )), (False, ('sub_dir', 'sub_file1')), (False, ('sub_dir', 'sub_file2')), (True, ('sub_empty_dir', )), (True, ('sub_dir', 'sub_sub_dir')), (False, ('sub_dir', 'sub_sub_dir', 'sub_sub_file')), ): path = os.path.join(dirname, *file_path) if is_dir: fsutil.makedirs(path) self.assertTrue(os.path.isdir(path)) else: fsutil.write_file(path, '') self.assertTrue(os.path.isfile(path)) fsutil.remove(dirname) self.assertFalse(os.path.exists(dirname))
def _write_wait(cont_write, cont_read, start_after, atomic): time.sleep(start_after) fsutil.write_file(fn, cont_write, atomic=atomic) if cont_read != fsutil.read_file(fn): assert_ok['ok'] = False
def test_get_sub_dirs(self): fsutil.makedirs('test_dir/sub_dir1') fsutil.makedirs('test_dir/sub_dir2') fsutil.write_file('test_dir/test_file', 'foo') sub_dirs = fsutil.get_sub_dirs('test_dir') self.assertListEqual(['sub_dir1', 'sub_dir2'], sub_dirs) fsutil.remove('test_dir')
def setUp(self): fsutil.makedirs(self.testSrc) for fn in self.src_fns: fsutil.write_file(os.path.join(self.testSrc, fn), fn) for d in self.arc_dirs: fsutil.makedirs(os.path.join(self.testArc, d))
def add_pids(cgroup_path, pids): task_file = os.path.join(cgroup_path, 'tasks') for pid in pids: try: fsutil.write_file(task_file, str(pid), fsync=False) except Exception as e: logger.info('failed to add pid: %s to file: %s, %s' % (str(pid), task_file, repr(e))) return
def test_remove_normal_file(self): fn = '/tmp/pykit-ut-fsutil-remove-file-normal' force_remove(fn) fsutil.write_file(fn, '', atomic=True) self.assertTrue(os.path.isfile(fn)) fsutil.remove(fn) self.assertFalse(os.path.exists(fn))
def set_cgroup(cgroup_path, conf): if 'weight' in conf: weight_file = os.path.join(cgroup_path, 'blkio.weight') weight = str(conf['weight']) fsutil.write_file(weight_file, weight, fsync=False) logger.info('write: %s to file: %s' % (weight, weight_file)) pids = conf.get('pids') if pids is None: return cgroup_util.add_pids(cgroup_path, pids) logger.info('add pids: %s to cgroup: %s' % (repr(pids), cgroup_path))
def set_cgroup(cgroup_path, conf): if 'share' not in conf: share = '1024' else: share = str(conf['share']) share_file = os.path.join(cgroup_path, 'cpu.shares') fsutil.write_file(share_file, share, fsync=False) logger.info('write: %s to file: %s' % (share, share_file)) pids = conf.get('pids') if pids is None: return cgroup_util.add_pids(cgroup_path, pids) logger.info('add pids: %s to cgroup: %s' % (repr(pids), cgroup_path))
def test_list_fns(self): fsutil.makedirs('test_dir/foo_dir') fsutil.write_file('test_dir/foo1', 'foo1') fsutil.write_file('test_dir/foo2', 'foo2') fsutil.write_file('test_dir/foo21', 'foo21') fsutil.write_file('test_dir/foo_dir/foo', 'foo') fsutil.write_file('test_dir/foo_dir/bar', 'bar') self.assertEqual(['foo1', 'foo2', 'foo21'], fsutil.list_fns('test_dir')) self.assertEqual(['foo2'], fsutil.list_fns('test_dir', pattern='2$')) self.assertEqual(['bar', 'foo'], fsutil.list_fns('test_dir/foo_dir')) self.assertEqual(['bar'], fsutil.list_fns('test_dir/foo_dir', pattern='^b')) fsutil.remove('test_dir')
def save_to_file(self, data): try: if not os.path.exists(self.data_file_dir): fsutil.makedirs(self.data_file_dir) except Exception as e: logger.error("mkdir: " + self.data_file_dir + " " + repr(e)) return -1 # data sent error try: content = "\n".join(data) fsutil.write_file(self.data_file, content, self.uid, self.gid) except Exception as e: logger.error("write data to " + self.data_file + " " + repr(e)) return -1 # data sent error
def test_offset_record_when_destory(self): expected = [x * 32 for x in 'qwertyuiop'] append_lines(self.fn, expected) cat_handle = fsutil.Cat(self.fn, strip=True) rst = [] for val in cat_handle.iterate(timeout=0): rst.append(val) self.assertEqual(expected, rst) # stat file was removed fsutil.remove(cat_handle.stat_path()) for val in cat_handle.iterate(timeout=0): rst.append(val) self.assertEqual(expected * 2, rst) # stat file was damaged fsutil.write_file(cat_handle.stat_path(), '{]') for val in cat_handle.iterate(timeout=0): rst.append(val) self.assertEqual(expected * 3, rst)
def test_read_write_file(self): fn = '/tmp/pykit-ut-rw-file' force_remove(fn) dd('write/read file') fsutil.write_file(fn, '123') self.assertEqual('123', fsutil.read_file(fn)) dd('write/read 3MB file') cont = '123' * (1024**2) fsutil.write_file(fn, cont) self.assertEqual(cont, fsutil.read_file(fn)) dd('write file with uid/gid') fsutil.write_file(fn, '1', uid=1, gid=1) stat = os.stat(fn) self.assertEqual(1, stat.st_uid) self.assertEqual(1, stat.st_gid) force_remove(fn)
def test_body(self): str1 = ''' 使命:Better Internet ,Better life 愿景:成为全球受人尊敬的科技公司;最具创新力;最佳雇主 未来白山的特质:渴求变革;让创新超越客户想象;全球化;真诚、并始终如一 信条:以用户为中心,其他一切水到渠成;专心将一件事做到极致;越快越好 ''' str2 = ''' 12343564343rfe fdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' str3 = ''' 838839938238838388383838 dddjjdkkksijdidhdhhhhddd djdjdfdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' fsutil.write_file( '/tmp/a.txt', str1 ) fsutil.write_file( '/tmp/b.txt', str2 ) def make_file_reader(): with open('/tmp/b.txt') as f: while True: buf = f.read(1024 * 1024) if buf == '': break yield buf def make_str_reader(): yield str3 str_reader = make_str_reader() str_size = len(str3) file_reader = make_file_reader() file_size = os.path.getsize('/tmp/b.txt') case = [ [ [ { 'name': 'metadata1', 'value': 'lvting', 'headers': {'Date': 'Dec, 20 Dec 2018 15:00:00 GMT'} } ], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata1', 'Date: Dec, 20 Dec 2018 15:00:00 GMT', '', 'lvting', '--{b}--'.format(b=self.test_multipart.boundary), ] ], [ [ { 'name': 'metadata2', 'value': [ open('/tmp/a.txt'), os.path.getsize('/tmp/a.txt') ], } ], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata2', '', str1, '--{b}--'.format(b=self.test_multipart.boundary), ] ], [ [ { 'name': 'metadata3', 'value': [file_reader, file_size, 'b.txt'], } ], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata3; ' + 'filename=b.txt', 'Content-Type: text/plain', '', str2, '--{b}--'.format(b=self.test_multipart.boundary), ] ], [ [ { 'name': 'metadata4', 'value': [str_reader, str_size], } ], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata4', '', str3, '--{b}--'.format(b=self.test_multipart.boundary), ] ], [ [ { 'name': 'metadata5', 'value': ['234ffhhif3323jjfjf3'] } ], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata5', '', '234ffhhif3323jjfjf3', '--{b}--'.format(b=self.test_multipart.boundary), ] ], ] for c in case: body = self.test_multipart.make_body_reader(c[0]) data = [] for x in body: data.append(x) self.assertEqual('\r\n'.join(c[1]), ''.join(data)) fsutil.remove('/tmp/a.txt') fsutil.remove('/tmp/b.txt')
def test_headers(self): case = [ [ { 'Content-Length': 998, 'Content-Type': 'aaplication/octet-stream' }, { 'Content-Length': 998, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) } ], [ { 'Content-Length': 1200 }, { 'Content-Length': 1200, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) } ], [ { 'Content-Type': 'application/octet-stream' }, { 'Content-Length': 1335, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) } ], [ None, { 'Content-Length': 1335, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) } ] ] str1 = ''' 使命:Better Internet ,Better life 愿景:成为全球受人尊敬的科技公司;最具创新力;最佳雇主 未来白山的特质:渴求变革;让创新超越客户想象;全球化;真诚、并始终如一 信条:以用户为中心,其他一切水到渠成;专心将一件事做到极致;越快越好 ''' str2 = ''' 12343564343rfe fdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' str3 = ''' 838839938238838388383838 dddjjdkkksijdidhdhhhhddd djdjdfdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' fsutil.write_file( '/tmp/a.txt', str1 ) fsutil.write_file( '/tmp/b.txt', str2 ) def make_file_reader(): with open('/tmp/b.txt') as f: while True: buf = f.read(1024 * 1024) if buf == '': break yield buf def make_str_reader(): yield str3 str_reader = make_str_reader() str_size = len(str3) file_reader = make_file_reader() file_size = os.path.getsize('/tmp/b.txt') fields = [ { 'name': 'metadata1', 'value': 'lvting', 'headers': {'Date': 'Dec, 20 Dec 2018 15:00:00 GMT'} }, { 'name': 'metadata2', 'value': [ open('/tmp/a.txt'), os.path.getsize('/tmp/a.txt'), ], 'headers': {'Content-Type': 'application/octet-stream'} }, { 'name': 'metadata3', 'value': [file_reader, file_size, 'b.txt'], }, { 'name': 'metadata4', 'value': [str_reader, str_size], }, { 'name': 'metadata5', 'value': ['234ffhhif3323jjfjf3'] }, ] for h in case: self.assertEqual( h[1], self.test_multipart.make_headers(fields, h[0]) ) fsutil.remove('/tmp/a.txt') fsutil.remove('/tmp/b.txt')
def store_progress(): fsutil.write_file(cnf['PROGRESS_FILE'], utfjson.dump(current_progress))
def test_body(self): str1 = ''' 使命:Better Internet ,Better life 愿景:成为全球受人尊敬的科技公司;最具创新力;最佳雇主 未来白山的特质:渴求变革;让创新超越客户想象;全球化;真诚、并始终如一 信条:以用户为中心,其他一切水到渠成;专心将一件事做到极致;越快越好 ''' str2 = ''' 12343564343rfe fdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' str3 = ''' 838839938238838388383838 dddjjdkkksijdidhdhhhhddd djdjdfdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' fsutil.write_file('/tmp/a.txt', str1) fsutil.write_file('/tmp/b.txt', str2) def make_file_reader(): with open('/tmp/b.txt') as f: while True: buf = f.read(1024 * 1024) if buf == '': break yield buf def make_str_reader(): yield str3 str_reader = make_str_reader() str_size = len(str3) file_reader = make_file_reader() file_size = os.path.getsize('/tmp/b.txt') case = [ [[{ 'name': 'metadata1', 'value': 'lvting', 'headers': { 'Date': 'Dec, 20 Dec 2018 15:00:00 GMT' } }], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata1', 'Date: Dec, 20 Dec 2018 15:00:00 GMT', '', 'lvting', '--{b}--'.format(b=self.test_multipart.boundary), ]], [[{ 'name': 'metadata2', 'value': [open('/tmp/a.txt'), os.path.getsize('/tmp/a.txt')], }], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata2', '', str1, '--{b}--'.format(b=self.test_multipart.boundary), ]], [[{ 'name': 'metadata3', 'value': [file_reader, file_size, 'b.txt'], }], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata3; ' + 'filename=b.txt', 'Content-Type: text/plain', '', str2, '--{b}--'.format(b=self.test_multipart.boundary), ]], [[{ 'name': 'metadata4', 'value': [str_reader, str_size], }], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata4', '', str3, '--{b}--'.format(b=self.test_multipart.boundary), ]], [[{ 'name': 'metadata5', 'value': ['234ffhhif3323jjfjf3'] }], [ '--{b}'.format(b=self.test_multipart.boundary), 'Content-Disposition: form-data; name=metadata5', '', '234ffhhif3323jjfjf3', '--{b}--'.format(b=self.test_multipart.boundary), ]], ] for c in case: body = self.test_multipart.make_body_reader(c[0]) data = [] for x in body: data.append(x) self.assertEqual('\r\n'.join(c[1]), ''.join(data)) fsutil.remove('/tmp/a.txt') fsutil.remove('/tmp/b.txt')
def reset_statistics(cgroup_path): reset_file = os.path.join(cgroup_path, 'blkio.reset_stats') fsutil.write_file(reset_file, '1', fsync=False)
def reset_statistics(cgroup_path): usage_file = os.path.join(cgroup_path, 'cpuacct.usage') fsutil.write_file(usage_file, '0', fsync=False)
def writeArc(self, size): fn = os.path.join(self.testArc, self.arc_dirs[-1], 'foo') content = os.urandom(size) fsutil.write_file(fn, content)
import os import sys from pykit import fsutil fn = sys.argv[1] fsutil.write_file(fn, 'boo') stat = os.stat(fn) os.write(1, '{uid},{gid}'.format(uid=stat.st_uid, gid=stat.st_gid))
def dump_state(): fsutil.write_file(cnf['STATE_FILE'], utfjson.dump(sync_state))
def test_calc_checksums(self): M = 1024**2 fn = '/tmp/pykit-ut-fsutil-calc_checksums' force_remove(fn) cases = ( ( '', { 'sha1': True, 'md5': True, 'crc32': True, 'sha256': True, 'block_size': M, 'io_limit': M, }, { 'sha1': 'da39a3ee5e6b4b0d3255bfef95601890afd80709', 'md5': 'd41d8cd98f00b204e9800998ecf8427e', 'crc32': '00000000', 'sha256': 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855', }, None, ), ( '', { 'md5': True, 'crc32': True, }, { 'sha1': None, 'md5': 'd41d8cd98f00b204e9800998ecf8427e', 'crc32': '00000000', 'sha256': None, }, None, ), ( '', {}, { 'sha1': None, 'md5': None, 'crc32': None, 'sha256': None, }, None, ), ( 'It 바로 とても 氣!', { 'sha1': True, 'md5': True, 'crc32': True, 'sha256': True, 'block_size': M, 'io_limit': M, }, { 'sha1': 'e22fa5446cb33d0c32221d89ee270dff23e32847', 'md5': '9d245fca88360be492c715253d68ba6f', 'crc32': '7c3becdb', 'sha256': '7327753b12db5c0dd090ad802c1c8ff44ea4cb447f3091d43cab371bd7583d9a', }, None, ), ( 'It 바로 とても 氣!', { 'sha1': False, 'md5': True, 'crc32': True, 'sha256': False, 'block_size': M, 'io_limit': M, }, { 'sha1': None, 'md5': '9d245fca88360be492c715253d68ba6f', 'crc32': '7c3becdb', 'sha256': None, }, None, ), ( 'It 바로 とても 氣!', { 'sha1': True, 'md5': False, 'crc32': False, 'sha256': True, }, { 'sha1': 'e22fa5446cb33d0c32221d89ee270dff23e32847', 'md5': None, 'crc32': None, 'sha256': '7327753b12db5c0dd090ad802c1c8ff44ea4cb447f3091d43cab371bd7583d9a', }, None, ), ( '!' * M * 10, { 'sha1': False, 'md5': False, 'crc32': False, 'sha256': False, 'block_size': M, 'io_limit': M, }, { 'sha1': None, 'md5': None, 'crc32': None, 'sha256': None, }, (0, 0.5), ), ( '!' * M * 10, { 'sha1': True, 'md5': True, 'crc32': True, 'sha256': True, 'block_size': M * 10, 'io_limit': M * 10, }, { 'sha1': 'c5430d624c498024d0f3371670227a201e910054', 'md5': '8f499b17375fc678c7256f3c0054db79', 'crc32': 'f0af209f', 'sha256': 'bd5263cc56b27fda9f86f41f6d2ec012eb60d757281003c363b88677c7dcc5e7', }, (1, 1.5), ), ( '!' * M * 10, { 'sha1': True, 'block_size': M, 'io_limit': M * 5, }, None, (2, 2.5), ), ( '!' * M * 10, { 'sha1': True, 'block_size': M, 'io_limit': -1, }, None, (0, 0.5), ), ) for cont, args, exp_checksums, min_time in cases: force_remove(fn) fsutil.write_file(fn, cont) t0 = time.time() checksums = fsutil.calc_checksums(fn, **args) spend_time = time.time() - t0 if exp_checksums is not None: for k in checksums: self.assertEqual( exp_checksums[k], checksums[k], 'except: {exp}, actuality: {act}'.format( exp=exp_checksums[k], act=checksums[k], )) if min_time is not None: self.assertTrue(min_time[0] < spend_time < min_time[1]) force_remove(fn)
class TestMultipart(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestMultipart, self).__init__(*args, **kwargs) self.test_multipart = None def setUp(self): self.test_multipart = httpmultipart.Multipart() def test_headers(self): case = [[{ 'Content-Length': 998, 'Content-Type': 'aaplication/octet-stream' }, { 'Content-Length': 998, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) }], [{ 'Content-Length': 1200 }, { 'Content-Length': 1200, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) }], [{ 'Content-Type': 'application/octet-stream' }, { 'Content-Length': 1335, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) }], [ None, { 'Content-Length': 1335, 'Content-Type': 'multipart/form-data; ' + 'boundary={b}'.format(b=self.test_multipart.boundary) } ]] str1 = ''' 使命:Better Internet ,Better life 愿景:成为全球受人尊敬的科技公司;最具创新力;最佳雇主 未来白山的特质:渴求变革;让创新超越客户想象;全球化;真诚、并始终如一 信条:以用户为中心,其他一切水到渠成;专心将一件事做到极致;越快越好 ''' str2 = ''' 12343564343rfe fdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' str3 = ''' 838839938238838388383838 dddjjdkkksijdidhdhhhhddd djdjdfdf4erguu38788894hf 12rfhfvh8876w91908777yfj ''' fsutil.write_file('/tmp/a.txt', str1) fsutil.write_file('/tmp/b.txt', str2) def make_file_reader(): with open('/tmp/b.txt') as f: while True: buf = f.read(1024 * 1024) if buf == '': break yield buf def make_str_reader(): yield str3 str_reader = make_str_reader() str_size = len(str3) file_reader = make_file_reader() file_size = os.path.getsize('/tmp/b.txt') fields = [ { 'name': 'metadata1', 'value': 'lvting', 'headers': { 'Date': 'Dec, 20 Dec 2018 15:00:00 GMT' } }, { 'name': 'metadata2', 'value': [ open('/tmp/a.txt'), os.path.getsize('/tmp/a.txt'), ], 'headers': { 'Content-Type': 'application/octet-stream' } }, { 'name': 'metadata3', 'value': [file_reader, file_size, 'b.txt'], }, { 'name': 'metadata4', 'value': [str_reader, str_size], }, { 'name': 'metadata5', 'value': ['234ffhhif3323jjfjf3'] }, ] for h in case: self.assertEqual(h[1], self.test_multipart.make_headers(fields, h[0])) fsutil.remove('/tmp/a.txt') fsutil.remove('/tmp/b.txt')