def _check_8192_wordlist(language, rejected):
print('Checking 8192 wordlist for ' + language)
file_name = path.join(base_dir, 'wordlists', language, 'wordlist-8192.txt')
if not os.path.isfile(file_name):
print(colorize('8192 wordlist not found for ' + language, 'yellow'))
return True
try:
with open(file_name, 'r') as f:
words = [line.rstrip('\n') for line in f.readlines()]
_check_for_rejected_words(words, rejected)
assert (len(words) == 8192)
assert (len(set(words)) == 8192)
_regex_test(words)
words_copy = sorted(copy.copy(words))
assert (words == words_copy)
print(colorize('Success for 8192 list for ' + language, 'green'))
return True
except:
print(colorize('Failure for 8192 list for ' + language, 'red'))
traceback.print_exc(file=sys.stdout)
return False
def _check_8192_wordlist(language, rejected):
print('Checking 8192 wordlist for ' + language)
file_name = path.join(base_dir, 'wordlists', language, 'wordlist-8192.txt')
if not os.path.isfile(file_name):
print(colorize('8192 wordlist not found for ' + language, 'yellow'))
return True
try:
with open(file_name, 'r') as f:
words = [line.rstrip('\n') for line in f.readlines()]
_check_for_rejected_words(words, rejected)
assert(len(words) == 8192)
assert(len(set(words)) == 8192)
_regex_test(words)
words_copy = sorted(copy.copy(words))
assert(words == words_copy)
print(colorize('Success for 8192 list for ' + language, 'green'))
return True
except:
print(colorize('Failure for 8192 list for ' + language, 'red'))
traceback.print_exc(file=sys.stdout)
return False
def _check_7776_wordlist(language, rejected):
print('Checking 7776 wordlist for ' + language)
try:
with open(path.join(base_dir, 'wordlists', language, 'wordlist.txt'),
'r') as f:
words = [line.rstrip('\n') for line in f.readlines()]
_check_for_rejected_words(words, rejected)
assert (len(words) == 7776)
assert (len(set(words)) == 7776)
_regex_test(words)
words_copy = sorted(words)
assert (words == words_copy)
_write_numbered_file(words, language)
print(colorize('Success for 7776 list for ' + language, 'green'))
return True
except:
print(colorize('Failure for 7776 list for ' + language, 'red'))
traceback.print_exc(file=sys.stdout)
return False
def _write_numbered_file(words, language):
print(colorize('Writing file for language ' + language, 'white'))
with open(path.join(base_dir, 'wordlists', language, 'wordlist-numbered.txt'), 'w') as f:
line_num = 0
for word in words:
f.write('{num}\t{word}\n'.format(num=_dice_num(line_num), word=word))
line_num += 1
print(colorize('Successfully wrote language ' + language, 'green'))
def _write_numbered_file(words, language):
print(colorize('Writing file for language ' + language, 'white'))
with open(
path.join(base_dir, 'wordlists', language,
'wordlist-numbered.txt'), 'w') as f:
line_num = 0
for word in words:
f.write('{num}\t{word}\n'.format(num=_dice_num(line_num),
word=word))
line_num += 1
print(colorize('Successfully wrote language ' + language, 'green'))
def finalize(args):
print(colorize('Checking wordlists...', 'white', True))
all_valid = True
for d in os.listdir(path.join(base_dir, 'wordlists')):
is_valid = _process_lang(d)
all_valid = all_valid and is_valid
if all_valid:
print(colorize('Success: all files valid and enumerated', 'green', True))
sys.exit(0)
else:
print(colorize('Failure: something bad happened', 'red', True))
sys.exit(1)
def finalize(args):
print(colorize('Checking wordlists...', 'white', True))
all_valid = True
for d in os.listdir(path.join(base_dir, 'wordlists')):
is_valid = _process_lang(d)
all_valid = all_valid and is_valid
if all_valid:
print(
colorize('Success: all files valid and enumerated', 'green', True))
sys.exit(0)
else:
print(colorize('Failure: something bad happened', 'red', True))
sys.exit(1)
def _process_lang(language):
print(colorize('Checking language ' + language, 'white'))
# TODO set locale based on 'language' var (for sorting)
rejected = _get_rejected_words(language)
result_7776 = _check_7776_wordlist(language, rejected)
result_8192 = _check_8192_wordlist(language, rejected)
return result_7776 and result_8192
def _process_lang(language):
print(colorize('Checking language ' + language, 'white'))
# TODO set locale based on 'language' var (for sorting)
rejected = _get_rejected_words(language)
result_7776 = _check_7776_wordlist(language, rejected)
result_8192 = _check_8192_wordlist(language, rejected)
return result_7776 and result_8192
def db_state(args):
engine.execute('vacuum')
statuses = db_session.query(WordList.language, WordList.state, func.count(WordList.word)) \
.group_by(WordList.language, WordList.state) \
.order_by(WordList.language, WordList.state).all()
# spacing: shitty, but good enough
print(colorize('Language\tState\t\tCount', 'white', True))
for status in statuses:
print('{lang}\t\t{state} \t{count}'.format(lang=status[0], state=status[1], count=status[2]))
def db_state(args):
engine.execute('vacuum')
statuses = db_session.query(WordList.language, WordList.state, func.count(WordList.word)) \
.group_by(WordList.language, WordList.state) \
.order_by(WordList.language, WordList.state).all()
# spacing: shitty, but good enough
print(colorize('Language\tState\t\tCount', 'white', True))
for status in statuses:
print('{lang}\t\t{state} \t{count}'.format(lang=status[0],
state=status[1],
count=status[2]))
def _check_7776_wordlist(language, rejected):
print('Checking 7776 wordlist for ' + language)
try:
with open(path.join(base_dir, 'wordlists', language, 'wordlist.txt'), 'r') as f:
words = [line.rstrip('\n') for line in f.readlines()]
_check_for_rejected_words(words, rejected)
assert(len(words) == 7776)
assert(len(set(words)) == 7776)
_regex_test(words)
words_copy = sorted(words)
assert(words == words_copy)
_write_numbered_file(words, language)
print(colorize('Success for 7776 list for ' + language, 'green'))
return True
except:
print(colorize('Failure for 7776 list for ' + language, 'red'))
traceback.print_exc(file=sys.stdout)
return False
def _get_rejected_words(language):
file_name = path.join(base_dir, 'wordlists', language, 'rejected.txt')
rejects = []
if not os.path.isfile(file_name):
print(colorize('No rejected words found for language ' + language, 'yellow'))
return rejects
with open(file_name, 'r') as f:
rejects = [line.rstrip('\n') for line in f.readlines()]
rejects_copy = sorted(rejects)
# because I'm pedantic and want everything sorted
assert(rejects == rejects_copy)
assert(len(set(rejects)) == len(rejects))
return rejects
def _get_rejected_words(language):
file_name = path.join(base_dir, 'wordlists', language, 'rejected.txt')
rejects = []
if not os.path.isfile(file_name):
print(
colorize('No rejected words found for language ' + language,
'yellow'))
return rejects
with open(file_name, 'r') as f:
rejects = [line.rstrip('\n') for line in f.readlines()]
rejects_copy = sorted(rejects)
# because I'm pedantic and want everything sorted
assert (rejects == rejects_copy)
assert (len(set(rejects)) == len(rejects))
return rejects