def _check_8192_wordlist(language, rejected): print('Checking 8192 wordlist for ' + language) file_name = path.join(base_dir, 'wordlists', language, 'wordlist-8192.txt') if not os.path.isfile(file_name): print(colorize('8192 wordlist not found for ' + language, 'yellow')) return True try: with open(file_name, 'r') as f: words = [line.rstrip('\n') for line in f.readlines()] _check_for_rejected_words(words, rejected) assert (len(words) == 8192) assert (len(set(words)) == 8192) _regex_test(words) words_copy = sorted(copy.copy(words)) assert (words == words_copy) print(colorize('Success for 8192 list for ' + language, 'green')) return True except: print(colorize('Failure for 8192 list for ' + language, 'red')) traceback.print_exc(file=sys.stdout) return False
def _check_8192_wordlist(language, rejected): print('Checking 8192 wordlist for ' + language) file_name = path.join(base_dir, 'wordlists', language, 'wordlist-8192.txt') if not os.path.isfile(file_name): print(colorize('8192 wordlist not found for ' + language, 'yellow')) return True try: with open(file_name, 'r') as f: words = [line.rstrip('\n') for line in f.readlines()] _check_for_rejected_words(words, rejected) assert(len(words) == 8192) assert(len(set(words)) == 8192) _regex_test(words) words_copy = sorted(copy.copy(words)) assert(words == words_copy) print(colorize('Success for 8192 list for ' + language, 'green')) return True except: print(colorize('Failure for 8192 list for ' + language, 'red')) traceback.print_exc(file=sys.stdout) return False
def _check_7776_wordlist(language, rejected): print('Checking 7776 wordlist for ' + language) try: with open(path.join(base_dir, 'wordlists', language, 'wordlist.txt'), 'r') as f: words = [line.rstrip('\n') for line in f.readlines()] _check_for_rejected_words(words, rejected) assert (len(words) == 7776) assert (len(set(words)) == 7776) _regex_test(words) words_copy = sorted(words) assert (words == words_copy) _write_numbered_file(words, language) print(colorize('Success for 7776 list for ' + language, 'green')) return True except: print(colorize('Failure for 7776 list for ' + language, 'red')) traceback.print_exc(file=sys.stdout) return False
def _write_numbered_file(words, language): print(colorize('Writing file for language ' + language, 'white')) with open(path.join(base_dir, 'wordlists', language, 'wordlist-numbered.txt'), 'w') as f: line_num = 0 for word in words: f.write('{num}\t{word}\n'.format(num=_dice_num(line_num), word=word)) line_num += 1 print(colorize('Successfully wrote language ' + language, 'green'))
def _write_numbered_file(words, language): print(colorize('Writing file for language ' + language, 'white')) with open( path.join(base_dir, 'wordlists', language, 'wordlist-numbered.txt'), 'w') as f: line_num = 0 for word in words: f.write('{num}\t{word}\n'.format(num=_dice_num(line_num), word=word)) line_num += 1 print(colorize('Successfully wrote language ' + language, 'green'))
def finalize(args): print(colorize('Checking wordlists...', 'white', True)) all_valid = True for d in os.listdir(path.join(base_dir, 'wordlists')): is_valid = _process_lang(d) all_valid = all_valid and is_valid if all_valid: print(colorize('Success: all files valid and enumerated', 'green', True)) sys.exit(0) else: print(colorize('Failure: something bad happened', 'red', True)) sys.exit(1)
def finalize(args): print(colorize('Checking wordlists...', 'white', True)) all_valid = True for d in os.listdir(path.join(base_dir, 'wordlists')): is_valid = _process_lang(d) all_valid = all_valid and is_valid if all_valid: print( colorize('Success: all files valid and enumerated', 'green', True)) sys.exit(0) else: print(colorize('Failure: something bad happened', 'red', True)) sys.exit(1)
def _process_lang(language): print(colorize('Checking language ' + language, 'white')) # TODO set locale based on 'language' var (for sorting) rejected = _get_rejected_words(language) result_7776 = _check_7776_wordlist(language, rejected) result_8192 = _check_8192_wordlist(language, rejected) return result_7776 and result_8192
def db_state(args): engine.execute('vacuum') statuses = db_session.query(WordList.language, WordList.state, func.count(WordList.word)) \ .group_by(WordList.language, WordList.state) \ .order_by(WordList.language, WordList.state).all() # spacing: shitty, but good enough print(colorize('Language\tState\t\tCount', 'white', True)) for status in statuses: print('{lang}\t\t{state} \t{count}'.format(lang=status[0], state=status[1], count=status[2]))
def _check_7776_wordlist(language, rejected): print('Checking 7776 wordlist for ' + language) try: with open(path.join(base_dir, 'wordlists', language, 'wordlist.txt'), 'r') as f: words = [line.rstrip('\n') for line in f.readlines()] _check_for_rejected_words(words, rejected) assert(len(words) == 7776) assert(len(set(words)) == 7776) _regex_test(words) words_copy = sorted(words) assert(words == words_copy) _write_numbered_file(words, language) print(colorize('Success for 7776 list for ' + language, 'green')) return True except: print(colorize('Failure for 7776 list for ' + language, 'red')) traceback.print_exc(file=sys.stdout) return False
def _get_rejected_words(language): file_name = path.join(base_dir, 'wordlists', language, 'rejected.txt') rejects = [] if not os.path.isfile(file_name): print(colorize('No rejected words found for language ' + language, 'yellow')) return rejects with open(file_name, 'r') as f: rejects = [line.rstrip('\n') for line in f.readlines()] rejects_copy = sorted(rejects) # because I'm pedantic and want everything sorted assert(rejects == rejects_copy) assert(len(set(rejects)) == len(rejects)) return rejects
def _get_rejected_words(language): file_name = path.join(base_dir, 'wordlists', language, 'rejected.txt') rejects = [] if not os.path.isfile(file_name): print( colorize('No rejected words found for language ' + language, 'yellow')) return rejects with open(file_name, 'r') as f: rejects = [line.rstrip('\n') for line in f.readlines()] rejects_copy = sorted(rejects) # because I'm pedantic and want everything sorted assert (rejects == rejects_copy) assert (len(set(rejects)) == len(rejects)) return rejects