def convert_to_utf8(src, output=None, override=False): src = upath.abspath(src) print(src) filename = os.path.basename(src) output = upath.abspath(output) if output else None dir_ = upath.abspath(os.path.dirname(src)) if override: dst = src else: if dir_ == output: dst = os.path.join(output, filename) else: dst = os.path.join(output or dir_, '_%s' % filename) if not os.path.exists(os.path.dirname(dst)): os.makedirs(os.path.dirname(dst)) with open(src, 'rb') as fr: data = fr.read() ct = chardet.detect(data) if ct['confidence'] > 0.90: encoding = ct['encoding'] else: encoding = UnicodeDammit(data).original_encoding encoding = fix_encoding(encoding) utf8_data = to_text(data, encoding=encoding) print('convert_to_utf8() encoding:%s file:%s' % (encoding, src)) with codecs.open(dst, 'w', 'utf-8') as fw: fw.write(utf8_data)
def convert_to_utf8(src, output=None, override=False): src = upath.abspath(src) print(src) filename = os.path.basename(src) output = upath.abspath(output) if output else None dir_ = upath.abspath(os.path.dirname(src)) if override: dst = src else: if dir_ == output: dst = os.path.join(output, filename) else: dst = os.path.join(output or dir_, '_%s' % filename) if not os.path.exists(os.path.dirname(dst)): os.makedirs(os.path.dirname(dst)) with open(src, 'rb') as fr: data = fr.read() ct = chardet.detect(data) if ct['confidence'] > 0.90: encoding = ct['encoding'] else: encoding = UnicodeDammit(data).original_encoding encoding = fix_encoding(encoding) utf8_data = to_text(data, encoding=encoding) print('convert_to_utf8() encoding:%s file:%s' % (encoding, src)) with codecs.open(dst, 'w', 'utf-8') as fw: fw.write(utf8_data)
def manual_convert(): root = upath.abspath(sys.argv[1]) if len(sys.argv) == 3 and sys.argv[2] == '--convert': dry_run = False else: dry_run = True log = codecs.open('../convert.log', 'w', 'utf-8') for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: try: src = os.path.join(curdir, name) print('Converting %s' % src) with open(src, 'rb') as f: data = f.read() dst = src if dry_run: dst = os.path.join(curdir, '_%s' % name) else: dst = src utf8_data = compat.to_text(data, encoding='gb18030') with codecs.open(dst, 'w', 'utf-8') as f: f.write(utf8_data) except Exception as e: traceback.print_exc() log.write(src)
def manual_convert(): root = upath.abspath(sys.argv[1]) if len(sys.argv) == 3 and sys.argv[2] == '--convert': dry_run = False else: dry_run = True log = codecs.open('../convert.log', 'w', 'utf-8') for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: try: src = os.path.join(curdir, name) print('Converting %s' % src) with open(src, 'rb') as f: data = f.read() dst = src if dry_run: dst = os.path.join(curdir, '_%s' % name) else: dst = src utf8_data = compat.to_text(data, encoding='gb18030') with codecs.open(dst, 'w', 'utf-8') as f: f.write(utf8_data) except Exception as e: traceback.print_exc() log.write(src)
def first_line_as_title(): root = upath.abspath(sys.argv[1]) if len(sys.argv) == 3 and sys.argv[2] == '-z': dry_run = False else: dry_run = True for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) if not os.path.isfile(src): continue print('Fix %s' % src) try: new_name = None with codecs.open(src, 'r', 'utf-8') as f: new_name = f.readline().strip() while not new_name or len(new_name) > 25: new_name = f.readline().strip() new_name = new_name.replace('标 题: ', '') if new_name and new_name != name: print('New name: %s' % new_name) if not dry_run: dst = os.path.join(curdir, '%s.txt' % new_name) shutil.move(src, dst) print('Rename to: %s' % dst) else: print('Skip %s' % src) except Exception as e: traceback.print_exc()
def first_line_as_title(): root = upath.abspath(sys.argv[1]) if len(sys.argv) == 3 and sys.argv[2] == '-z': dry_run = False else: dry_run = True for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) if not os.path.isfile(src): continue print('Fix %s' % src) try: new_name = None with codecs.open(src, 'r', 'utf-8') as f: new_name = f.readline().strip() while not new_name or len(new_name) > 25: new_name = f.readline().strip() new_name = new_name.replace('标 题: ', '') if new_name and new_name != name: print('New name: %s' % new_name) if not dry_run: dst = os.path.join(curdir, '%s.txt' % new_name) shutil.move(src, dst) print('Rename to: %s' % dst) else: print('Skip %s' % src) except Exception as e: traceback.print_exc()
def smart_convert(): arg = upath.abspath(sys.argv[1]) if upath.isfile(arg): convert_to_utf8(arg) else: for name in upath.listdir(arg): convert_to_utf8(os.path.join(arg, name), override=True)
def smart_convert(): arg = upath.abspath(sys.argv[1]) if upath.isfile(arg): convert_to_utf8(arg) else: for name in upath.listdir(arg): convert_to_utf8(os.path.join(arg, name), override=True)
def move_noname_files(): root = upath.abspath(sys.argv[1]) output = upath.abspath(sys.argv[2]) dry_run = not (len(sys.argv) == 4 and sys.argv[3] == '-z') if not os.path.exists(output): os.makedirs(output) p = re.compile(r'[a-zA-Z0-9]+\.txt') for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) dst = os.path.join(output, name) # print('Processing %s' % src) if not os.path.isfile(src): continue if src == dst: continue if re.match(p, name): if not dry_run: print('Moving %s %s' % (src, dst)) shutil.move(src, dst)
def move_some_files(): root = upath.abspath(sys.argv[1]) output = upath.abspath(sys.argv[2]) dry_run = not (len(sys.argv) == 4 and sys.argv[3] == '-z') if not os.path.exists(output): os.makedirs(output) for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) dst = os.path.join(output, name) # print('Processing %s' % src) if not os.path.isfile(src): continue if src == dst: continue file_size = os.path.getsize(src) if file_size < 30000: if not dry_run: print('Moving %s %s size=%s' % (src, dst, file_size)) shutil.move(src, dst)
def move_some_files(): root = upath.abspath(sys.argv[1]) output = upath.abspath(sys.argv[2]) dry_run = not (len(sys.argv) == 4 and sys.argv[3] == '-z') if not os.path.exists(output): os.makedirs(output) for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) dst = os.path.join(output, name) # print('Processing %s' % src) if not os.path.isfile(src): continue if src == dst: continue file_size = os.path.getsize(src) if file_size < 30000: if not dry_run: print('Moving %s %s size=%s' % (src, dst, file_size)) shutil.move(src, dst)
def move_noname_files(): root = upath.abspath(sys.argv[1]) output = upath.abspath(sys.argv[2]) dry_run = not (len(sys.argv) == 4 and sys.argv[3] == '-z') if not os.path.exists(output): os.makedirs(output) p = re.compile(r'[a-zA-Z0-9]+\.txt') for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) dst = os.path.join(output, name) # print('Processing %s' % src) if not os.path.isfile(src): continue if src == dst: continue if re.match(p, name): if not dry_run: print('Moving %s %s' % (src, dst)) shutil.move(src, dst)
def move_by_pattern(): root = upath.abspath(sys.argv[1]) output = upath.abspath(sys.argv[2]) re_str = compat.to_text(sys.argv[3]) pattern = re.compile(re_str) dry_run = not (len(sys.argv) == 5 and sys.argv[4] == '-z') # output = os.path.join(output,re_str) if not os.path.exists(output): os.makedirs(output) for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) dst = os.path.join(output, name) # print('Processing %s' % src) if not os.path.isfile(src): continue if src == dst: continue if re.search(pattern, name): if not dry_run: print('Moving %s %s' % (src, dst)) shutil.move(src, dst) else: print('Demo %s %s' % (src, dst))
def move_by_pattern(): root = upath.abspath(sys.argv[1]) output = upath.abspath(sys.argv[2]) re_str = compat.to_text(sys.argv[3]) pattern = re.compile(re_str) dry_run = not (len(sys.argv) == 5 and sys.argv[4] == '-z') # output = os.path.join(output,re_str) if not os.path.exists(output): os.makedirs(output) for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) dst = os.path.join(output, name) # print('Processing %s' % src) if not os.path.isfile(src): continue if src == dst: continue if re.search(pattern, name): if not dry_run: print('Moving %s %s' % (src, dst)) shutil.move(src, dst) else: print('Demo %s %s' % (src, dst))
def remove_links(): root = upath.abspath(sys.argv[1]) p1 = re.compile(RE_URL) p2 = re.compile(r'[A-z0-9]+\.[A-z0-9]+\.[A-z0-9]+') for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) if not os.path.isfile(src): continue _, ext = os.path.splitext(name) if not ext or ext.lower() != '.txt': continue with codecs.open(src, 'r', 'utf8') as f: content = f.read() content2 = re.sub(p2, '', content) if len(content2) < len(content): print('Processing %s (%s %s)' % (src, len(content), len(content2))) with codecs.open(src, 'w', 'utf8') as f: f.write(content2)
def remove_invalid_chars(): root = upath.abspath(sys.argv[1]) pattern = re.compile('[\u4E00-\u9FA5,.]+') # 中文 invalid = re.compile('\uE4C6+') # \xee\x93\x86 U+E4C6 方块问号 for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) if not os.path.isfile(src): continue _, ext = os.path.splitext(name) if not ext or ext.lower() != '.txt': continue with codecs.open(src, 'r', 'utf8') as f: content = f.read() content2 = content.replace('\uE4C6','') if len(content2) < len(content): print('Processing %s (%s %s)' % (src, len(content), len(content2))) with codecs.open(src, 'w', 'utf8') as f: f.write(content2)
def remove_invalid_chars(): root = upath.abspath(sys.argv[1]) pattern = re.compile('[\u4E00-\u9FA5,.]+') # 中文 invalid = re.compile('\uE4C6+') # \xee\x93\x86 U+E4C6 方块问号 for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) if not os.path.isfile(src): continue _, ext = os.path.splitext(name) if not ext or ext.lower() != '.txt': continue with codecs.open(src, 'r', 'utf8') as f: content = f.read() content2 = content.replace('\uE4C6', '') if len(content2) < len(content): print('Processing %s (%s %s)' % (src, len(content), len(content2))) with codecs.open(src, 'w', 'utf8') as f: f.write(content2)
def remove_links(): root = upath.abspath(sys.argv[1]) p1 = re.compile(RE_URL) p2 = re.compile(r'[A-z0-9]+\.[A-z0-9]+\.[A-z0-9]+') for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: src = os.path.join(curdir, name) if not os.path.isfile(src): continue _, ext = os.path.splitext(name) if not ext or ext.lower() != '.txt': continue with codecs.open(src, 'r', 'utf8') as f: content = f.read() content2 = re.sub(p2, '', content) if len(content2) < len(content): print('Processing %s (%s %s)' % (src, len(content), len(content2))) with codecs.open(src, 'w', 'utf8') as f: f.write(content2)
def enca_convert(): root = upath.abspath(sys.argv[1]) if len(sys.argv) == 3 and sys.argv[2] == '-z': dry_run = False else: dry_run = True log = codecs.open('../enca.log', 'w', 'utf-8') for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: try: f = os.path.join(curdir, name) if dry_run: cmd = ['enca', f] else: cmd = 'iconv -f GB18030 -t UTF-8 -c'.split() cmd.append(f) print(cmd) r = subprocess.check_output(cmd) r = compat.to_text(r) print('%s - %s' % (f, r)) except Exception as e: traceback.print_exc() log.write(f)
def enca_convert(): root = upath.abspath(sys.argv[1]) if len(sys.argv) == 3 and sys.argv[2] == '-z': dry_run = False else: dry_run = True log = codecs.open('../enca.log', 'w', 'utf-8') for curdir, subdirs, filenames in os.walk(root, topdown=True): for name in filenames: try: f = os.path.join(curdir, name) if dry_run: cmd = ['enca', f] else: cmd = 'iconv -f GB18030 -t UTF-8 -c'.split() cmd.append(f) print(cmd) r = subprocess.check_output(cmd) r = compat.to_text(r) print('%s - %s' % (f, r)) except Exception as e: traceback.print_exc() log.write(f)
def move_from_subdirs(): root = upath.abspath(sys.argv[1]) if len(sys.argv) == 3 and sys.argv[2] == '-z': dry_run = False else: dry_run = True log = codecs.open('../fix_title.log', 'w', 'utf-8') print(root) for curdir, subdirs, filenames in os.walk(root, topdown=True): if curdir == root: continue for name in filenames: src = os.path.join(curdir, name) if not os.path.isfile(src): continue dst = os.path.join(root, name) print('File %s -> %s' % (src, dst)) try: if not dry_run: shutil.move(src, dst) print('Move %s -> %s' % (src, dst)) except Exception as e: traceback.print_exc() log.write(src)
def move_from_subdirs(): root = upath.abspath(sys.argv[1]) if len(sys.argv) == 3 and sys.argv[2] == '-z': dry_run = False else: dry_run = True log = codecs.open('../fix_title.log', 'w', 'utf-8') print(root) for curdir, subdirs, filenames in os.walk(root, topdown=True): if curdir == root: continue for name in filenames: src = os.path.join(curdir, name) if not os.path.isfile(src): continue dst = os.path.join(root, name) print('File %s -> %s' % (src, dst)) try: if not dry_run: shutil.move(src, dst) print('Move %s -> %s' % (src, dst)) except Exception as e: traceback.print_exc() log.write(src)