def work(self, site):
_, hostname, _ = get_host(site)
conn = utils.http_req(site)
item = {
"site": site,
"hostname": hostname,
"ip":"",
"title": utils.get_title(conn.content),
"status": conn.status_code,
"headers": utils.get_headers(conn),
"http_server": conn.headers.get("Server", ""),
"body_length": len(conn.content),
"finger": [],
"favicon": fetch_favicon(site)
}
domain_parsed = utils.domain_parsed(hostname)
if domain_parsed:
item["fld"] = domain_parsed["fld"]
ips = utils.get_ip(hostname)
if ips:
item["ip"] = ips[0]
else:
item["ip"] = hostname
self.site_info_list.append(item)
if conn.status_code == 301 or conn.status_code == 302:
url_302 = urljoin(site, conn.headers.get("Location", ""))
if url_302 != site and url_302.startswith(site):
self.work(url_302)
def copy_to_form(scenario, index):
global database
print('keys in database:', database.keys())
selected = database[scenario]['data'][int(index)]
return render_template('scenario.html',
title=utils.get_title(scenario),
scenario=scenario,
data=database[scenario]['data'],
columns=database[scenario]['columns'],
selected=selected)
def test_fetch_fingerprint(self):
site = "https://www.baidu.com/"
conn = utils.http_req(site)
headers = utils.get_headers(conn)
title = utils.get_title(conn.content)
finger_list = [
{
"name": "百度测试",
"rule": {
"html": [
"百度"
],
"title": [],
"headers": [],
"favicon_hash": []
}
},
{
"name": "百度测试2",
"rule": {
"html": [],
"title": ["百度222222", "百度"],
"headers": [],
"favicon_hash": []
}
},
{
"name": "百度测试3",
"rule": {
"html": [],
"title": [],
"headers": ["TTTBAIDUIDTTT", "BAIDUID"],
"favicon_hash": []
}
},
{
"name": "百度测试4",
"rule": {
"html": [],
"title": [],
"headers": [],
"favicon_hash": [789, 123456]
}
}
]
finger_list.extend(load_fingerprint())
result = fetch_fingerprint(content=conn.content, headers=headers,
title=title, favicon_hash=789, finger_list=finger_list)
self.assertTrue(len(result) >= 4)
self.assertTrue(result[0] == finger_list[0]["name"])
self.assertTrue(result[3] == finger_list[3]["name"])
def load_scenario(scenario):
global database
data, columns = models.get_data(scenario)
if scenario not in database.keys():
database[scenario] = {'data': data, 'columns': columns}
return render_template('scenario.html',
title=utils.get_title(scenario),
scenario=scenario,
data=data,
columns=columns,
selected=None)
def enviar_exemplo(scenario):
features = utils.get_features(request.form)
url = 'http://localhost:5001/predict?model={}&features={}'.format(
model_maps[scenario], features)
print('url', url)
resultado = requests.get(url).json()[0]
return render_template('scenario.html',
title=utils.get_title(scenario),
scenario=scenario,
data=database[scenario]['data'],
columns=database[scenario]['columns'],
prediction=resultado)
def title(self) -> str:
if self._title is None:
self._title = utils.get_title(self.content)
return self._title