def csrf_substitude_code_in_text(self, target,
flow: mitmproxy.http.HTTPFlow):
if target in flow.response.text:
self.logger.write_info("csrf on value in response text")
if self.browserID in flow.request.headers.keys():
key_value = extract_code_from_content(flow.response.text,
target)
self.logger.write_info("[target] " + target + " " + key_value)
self.logger.write_file(self.RAM + target, key_value)
self.logger.write_info("Successfully save key value to " +
self.RAM + target)
flow.kill()
self.logger.write_info("kill flow")
else:
self.logger.write_info(
"[ORIGIN " + target + "] " +
extract_code_from_content(flow.response.text, target))
with open(self.RAM + target, 'r+') as f:
key_value = f.readlines()[0]
self.logger.write_info("[CHANGE " + target + "] " + key_value)
substitute_access_token_in_text(flow, target, key_value)
self.logger.write_info("If succeed? " +
str(key_value in flow.response.text))
return True
return False
def csrf_substitude_code_in_location(self, target,
flow: mitmproxy.http.HTTPFlow):
if "location" in flow.response.headers.keys():
location = flow.response.headers["location"]
if target not in location:
return False
self.logger.write_info("csrf on key value in location header")
self.logger.write_info("[location]: " + location)
if self.browserID in flow.request.headers.keys():
key_value = extract_code_from_content(location, target)
self.logger.write_info("[target] " + target + " " + key_value)
self.logger.write_file(self.RAM + target, key_value)
self.logger.write_info("Successfully save key value to " +
self.RAM + target)
flow.kill()
self.logger.write_info("kill flow")
else:
self.logger.write_info(
"[ORIGIN " + target + "] " +
extract_code_from_content(location, target))
with open(self.RAM + target, 'r+') as f:
key_value = f.readlines()[0]
self.logger.write_info("[CHANGE " + target + "] " + key_value)
flow.response.headers["location"] = substitute_code(
location, target, key_value)
self.logger.write_info("If succeed? " + str(
key_value in flow.response.headers["location"]))
return True
return False
def request(self, flow: mitmproxy.http.HTTPFlow):
uri = flow.request.path
if uri.find("a_new_task_here_rebirth") != -1:
ref = flow.request.path
pos = ref.find("?url=")
if (pos != -1):
ref = ref[pos + 5:]
self.task = ref
self.task = urllib.parse.unquote(self.task)
flow.response = mitmproxy.http.HTTPResponse.make(404)
#print("bingo")
return
if uri.find("a_new_req_here_rebirth") != -1:
ref = flow.request.path
pos = ref.find("?url=")
if (pos != -1):
ref = ref[pos + 5:]
self.req = ref
self.req = urllib.parse.unquote(self.req)
flow.response = mitmproxy.http.HTTPResponse.make(404)
#print("bingo")
return
# if self.task != "":
# print(self.task)
# flow.request.headers["Referer"] = self.task
url = flow.request.host + uri
for pattern in self.patterns:
if re.search(pattern["pattern"], url):
self.db.insert(self.task, 1, pattern["service"], url, self.req)
break
return
def test_url_parameters(flow: mitmproxy.http.HTTPFlow, pattern: str = "*"):
global simplified_url
# check legality
if flow.request.pretty_url.find(pattern) == -1 and pattern != "*":
return
# check if this is a replay flow
if "test_url_parameters" in flow.request.headers.keys():
if flow.request.headers["test_url_parameters"] == "true":
# This is a replay flow. It needs to be test with its response
simplified_url[flow.request.headers["flow_id"]].judge_parameter(flow)
flow.intercept()
time.sleep(4)
flow.resume()
return
origin_url = flow.request.pretty_url
# if no query in url, return
if origin_url.find("?") == -1:
return
# block this flow
flow.intercept()
# extract the parameters in url
url = origin_url.split("?")
assert len(url) == 2
parameters = url[1].split("&")
flow_id = str(hash(flow))
simplified_url[flow_id] = Simplified_URL(flow, parameters)
# try removing each parameters in url
# replay and see the change of response
for i, val in enumerate(parameters):
# compose fake url
fake_url = origin_url
index = fake_url.find(val)
if not fake_url.find("&", index) == -1:
fake_url = fake_url[:index] + fake_url[index+len(val)+1:]
else:
fake_url = fake_url[:index] + fake_url[index+len(val):]
if fake_url.endswith("&"):
fake_url = fake_url[:-1]
# ctx.log.info("fake_url: " + fake_url)
# replay the flow with fake url
fake_flow = flow.copy()
fake_flow.request.headers["test_url_parameters"] = "true"
fake_flow.request.headers["deleted"] = val
fake_flow.request.headers["flow_id"] = flow_id
fake_flow.request.url = fake_url
replay(fake_flow)
time.sleep(3)
ctx.log.info("origin_url: " + origin_url)
surl = simplified_url[flow_id].summary()
ctx.log.info("simplified_url: " + surl)
flow.resume() # release the block
return surl
def test_post_form(flow: mitmproxy.http.HTTPFlow, pattern: str = "*"):
global simplified_form
# check legality
if flow.request.pretty_url.find(pattern) == -1 and pattern != "*":
return
if flow.request.method != "POST":
return
if is_relpay_flow("test_post_form", flow, simplified_form):
return
origin_form = flow.request.text
# block this flow
flow.intercept()
# extract the parameters in form
parameters = origin_form.split("&")
flow_id = str(hash(flow))
simplified_form[flow_id] = Simplified_Form(flow, parameters)
# try removing each parameters in form
# replay and see the change of response
for i, val in enumerate(parameters):
# compose fake form
fake_form = origin_form
index = fake_form.find(val)
if not fake_form.find("&", index) == -1:
fake_form = fake_form[:index] + fake_form[index + len(val) + 1:]
else:
fake_form = fake_form[:index] + fake_form[index + len(val):]
if fake_form.endswith("&"):
fake_form = fake_form[:-1]
ctx.log.info("fake_form: " + fake_form)
# replay the flow with fake form
fake_flow = flow.copy()
fake_flow.request.headers["test_post_form"] = "true"
fake_flow.request.headers["deleted"] = val
fake_flow.request.headers["flow_id"] = flow_id
fake_flow.request.text = fake_form
replay(fake_flow)
time.sleep(6)
ctx.log.info("origin_form: " + origin_form)
sform = simplified_form[flow_id].summary()
del simplified_form[flow_id]
ctx.log.info("simplified_form: " + sform)
flow.resume() # release the block
return sform
def __init__(self, flow: mitmproxy.http.HTTPFlow, parameters: list):
self.origin_flow = flow.copy()
self.finished = False
self.parameters = {}
self.response: mitmproxy.http.HTTPResponse = None
for val in parameters:
self.parameters[val] = (False, True)
def __init__(self, flow: mitmproxy.http.HTTPFlow, parameters: list):
self.origin_flow = flow.copy()
self.origin_form = flow.request.text
self.origin_response = flow.response
self.form_parameters = {}
for val in parameters:
self.form_parameters[val] = False
def request(flow: mitmproxy.http.HTTPFlow):
# TODO: Just send request kwargs rather than serialized `PreparedRequest` object.
r = flow.request
req = requests.Request(
method=r.method,
url=r.pretty_url,
headers=r.headers,
cookies=r.cookies,
params=r.query,
data=r.raw_content,
)
prepped = req.prepare()
serialized_req = pickle.dumps(prepped)
flow.request = flow.request.make(
"POST",
url=scf_server,
content=b64encode(serialized_req),
headers={
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, compress",
"Accept-Language": "en-us;q=0.8",
"Cache-Control": "max-age=0",
"User-Agent":
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"Connection": "close",
"Host": urlparse(scf_server).netloc,
"SCF-Token": SCF_TOKEN,
},
)
def response_hello_world(flow: mitmproxy.http.HTTPFlow):
if flow.request.pretty_url.endswith("gitlab.com/users/sign_in"):
flow.response = mitmproxy.http.HTTPResponse.make(
200,
"<html><body>hello world</body></html>",
{"content-type": "text/html"},
)
def request(flow: mitmproxy.http.HTTPFlow) -> None:
scf_servers = choice(scf_servers)
r = flow.request
data = {
"url": r.pretty_url,
"method": r.method,
"headers": dict(r.headers),
"cookies": dict(r.cookies),
"params": dict(r.query),
"data": b64encode(r.raw_content).decode("ascii"),
}
flow.request = flow.request.make(
"POST",
url=scf_servers,
content=json.dumps(data),
headers={
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"Accept-Encoding": "gzip, deflate, compress",
"Accept-Language": "en-us;q=0.8",
"Cache-Control": "max-age=0",
"Connection": "close",
"user-agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36",
"host": urlparse(scf_servers).netloc,
"SCF-Token": SCF_TOKRN,
})
def request(flow: mitmproxy.http.HTTPFlow):
scf_server = choice(scf_servers)
r = flow.request
data = {
"method": r.method,
"url": r.pretty_url,
"headers": dict(r.headers),
"cookies": dict(r.cookies),
"params": dict(r.query),
"data": b64encode(r.raw_content).decode("ascii"),
}
flow.request = flow.request.make(
"POST",
url=scf_server,
content=json.dumps(data),
headers={
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, compress",
"Accept-Language": "en-us;q=0.8",
"Cache-Control": "max-age=0",
"User-Agent":
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"Connection": "close",
"Host": urlparse(scf_server).netloc,
"SCF-Token": SCF_TOKEN,
},
)
def response(flow: mitmproxy.http.HTTPFlow) -> None:
if flow.response.status_code != 200:
mitmproxy.ctx.log("[-] %s" % flow.response.status_code)
if flow.response.status_code == 401:
flow.response.headers = Headers(
content_type="text/html;chartset=utf-8")
return
if flow.response.status_code == 433:
flow.response.headers = Headers(
content_type="text/html;chartset=utf-8")
flow.response.text = "<html><body><h1>403 Forbidden</h1><p>You have been blocked by Cloudflare.</p></body></html>"
return
if flow.response.status_code == 200:
body = flow.response.content.decode("utf-8")
resp = pickle.loads(b64decode(body))
r = flow.response.make(
status_code=resp.status_code,
content=b64decode(resp.data),
headers=dict(resp.headers),
)
flow.response = r
def http_connect(self, flow: mitmproxy.http.HTTPFlow):
# LAYER 1 : BLACKLIST BLOCK ------------------------
fo = open("blacklist.txt", "r+")
line = fo.readline()
while line:
ctx.log.info("now comparing:" +
line.replace("\n", "").replace("\r", ""))
if flow.request.host == line.replace("\n", "").replace(
"\r", "").strip() and line.strip() != "":
flow.response = http.HTTPResponse.make(404)
ctx.log.info(line + " BAN.")
return
line = fo.readline()
fo.close()
# LAYER 2 : AUTONOMOUS BLOCK -----------------------
if self.a == "0": # all block
flow.response = http.HTTPResponse.make(404)
#flow.request.host = "www.bing.cn"
ctx.log.info("ALL BLOCK CHAIN PERFORMED.")
if self.a == "1" or self.a == "2": # only block black
# fo = open("blacklist.txt","r+")
# line = purify(fo.readline())
# while line:
# ctx.log.info("now matching:"+line)
# if flow.request.host == line:
# ctx.log.info("matched.block")
# flow.response = http.HTTPResponse.make(404)
# line = purify(fo.readline())
# fo.close()
pass
if self.a == "2": # smart block + black
pass
if self.a == "3": # only white
fo = open("whitelist.txt", "r+")
line = purify(fo.readline())
while line:
ctx.log.info("now granting:" + line)
if flow.request.host == line:
ctx.log.info("ACCESS GRANTED.")
return
line = purify(fo.readline())
flow.response = http.HTTPResponse.make(404)
ctx.log.info("ACCORDING TO POLICY 3 , NOW DENIED.")
fo.close()
ctx.log.info("ORZ -------------------------------")
def csrf_substitude_header(self, target, flow: mitmproxy.http.HTTPFlow):
if target in flow.response.headers.keys():
self.logger.write_info("csrf change a whole header value")
value = flow.response.headers[target]
if self.browserID in flow.request.headers.keys():
key_value = value
self.logger.write_info("[target] " + target + " " + key_value)
self.logger.write_file(self.RAM + target, key_value)
self.logger.write_info("Successfully save key value to " +
self.RAM + target)
flow.kill()
self.logger.write_info("kill flow")
else:
self.logger.write_info("[ORIGIN " + target + "] " + value)
with open(self.RAM + target, 'r+') as f:
key_value = f.readlines()[0]
self.logger.write_info("[CHANGE " + target + "] " + key_value)
flow.response.headers[target] = key_value
self.logger.write_info("If succeed? " + str(
key_value in flow.response.headers[target]))
def request(self, flow: mitmproxy.http.HTTPFlow):
"""
The full HTTP request has been read.
"""
if "https://stock.xueqiu.com/v5/stock/batch/quote.json" in flow.request.url and \
"x=" in flow.request.url:
# print("雪球"*10)
print(flow)
with open("../datas/quote.json", encoding="utf-8") as f:
flow.response = http.HTTPResponse.make(200, f.read())
print("request done")
def change_response_immediately(self, flow: mitmproxy.http.HTTPFlow):
if self.is_request_match(flow):
modified_response = open(self.error_response_file, "r").read()
flow.response = http.HTTPResponse.make(
200, # (optional) status code
modified_response, # (optional) content
{
"access-control-allow-origin": "https://www.tokopedia.com",
"access-control-allow-credentials": "true",
"content-type": "application/json",
"access-control-allow-headers": "Content-type, Fingerprint-Data, Fingerprint-Hash, x-user-id, Webview-App-Version, Redirect, Access-Control-Allow-Origin, Content-MD5, Tkpd-UserId, X-Tkpd-UserId, Tkpd-SessionId, X-Device, X-Source, X-Method, X-Date, Authorization, Accounts-Authorization, flight-thirdparty, x-origin, Cshld-SessionID, X-Mitra-Device, x-tkpd-akamai, x-tkpd-lite-service, x-ga-id, Akamai-Bot, x-tkpd-app-name, x-tkpd-clc, x-return-hmac-md5"
}
)
def request(flow: mitmproxy.http.HTTPFlow):
"""
The full HTTP request has been read.
"""
global log_file
host = checker.check_host(flow)
if host:
if checker.check_TLS(flow):
logger.write(log_file, \
"[TLS] " + flow.request.pretty_url)
if "test.xxx" in flow.request.host:
flow.kill()
# csrf
global access_token
target = "fb_access_token="
if target in flow.request.pretty_url:
if "longming" in flow.request.headers.keys():
# access_token = csrf.extract_code(flow, target)
access_token = flow.request.pretty_url
logger.write_info(log_file, "[TOKEN] " + access_token)
assert access_token
logger.write_file("RAM/access_token", access_token)
flow.kill()
else:
l = os.listdir('RAM')
while not l:
time.sleep(1)
l = os.listdir('RAM')
logger.write_info(
log_file, "[ORIGIN TOKEN] " + csrf.extract_code(flow, target))
with open('RAM/' + l[0], 'r+') as f:
access_token = f.readlines()[0]
logger.write_info(log_file, "[CHANGE TOKEN] " + access_token)
assert access_token
# assert csrf.csrf_request(flow, target, access_token)
flow.request.url = access_token
def request(self, flow: mitmproxy.http.HTTPFlow):
"""
The full HTTP request has been read.
"""
#匹配规则
if "https://stock.xueqiu.com/v5/stock/batch/quote.json?_t" in flow.request.url and "x=" in flow.request.url:
with open("quote.json", encoding="utf-8") as f:
flow.response = http.HTTPResponse.make(
# 状态码
200,
# 响应体，传入数据格式为str
f.read(),
# 响应头
)
def request(self, flow: mitmproxy.http.HTTPFlow):
'''
网络层
dns查询之后
'''
url = flow.request.scheme + '://' + \
flow.request.host + \
flow.request.path.split('?')[0] + ' ' + flow.request.method
db = sqlite3.connect(database)
cursor = db.cursor()
sql = f"select * from Mock1 where url='{url}' and status='1'"
print(f'拦截{url}到本地')
result = cursor.execute(sql)
js = [i for i in cursor.execute(sql)]
if len(js) > 0:
result = json.loads(parse.unquote(js[0][1]))
# flow.response = result['data']['response']
response = result['data'].get('response', None)
if not response:
return None
headers = {}
try:
for header in response['headers']:
headers[header[0]] = header[1]
except:
pass
content_type = headers.get('content-type', None) or headers.get(
'Content-Type', None)
html = response['html']
if 'image' in content_type or 'video' in content_type:
html = base64.b64decode(html.encode())
flow.response = mitmproxy.http.HTTPResponse.make(
response['status_code'] or 200, # (optional) status code
html, # (optional) content
headers # (optional) headers
)
cursor.close()
db.close()
def response(flow: mitmproxy.http.HTTPFlow):
if flow.response.status_code != 200:
mitmproxy.ctx.log.warn("Error")
if flow.response.status_code == 401:
flow.response.headers = Headers(content_type="text/html;charset=utf-8")
return
if flow.response.status_code == 433:
flow.response.headers = Headers(content_type="text/html;charset=utf-8")
flow.respons.content = "<html><body>操作已超过云函数服务最大时间限制，可<a href='https://console.cloud.tencent.com/workorder/category'>提交工单</a>申请提升超时限制</body></html>",
return
if flow.response.status_code == 200:
body = flow.response.content.decode("utf-8")
resp = pickle.loads(b64decode(body))
r = flow.response.make(
status_code=resp.status_code,
headers=dict(resp.headers),
content=resp.content,
)
flow.response = r
def response(flow: mitmproxy.http.HTTPFlow):
if flow.response.status_code != 200:
mitmproxy.ctx.log.warn("Error")
if flow.response.status_code == 401:
flow.response.headers = Headers(content_type="text/html;charset=utf-8")
return
if flow.response.status_code == 433:
flow.response.headers = Headers(content_type="text/html;charset=utf-8")
flow.response.content = (
"<html><body>操作已超过云函数服务最大时间限制，可在函数配置中修改执行超时时间</body></html>", )
return
if flow.response.status_code == 200:
body = flow.response.content.decode("utf-8")
resp = pickle.loads(b64decode(body))
r = flow.response.make(
status_code=resp.status_code,
headers=dict(resp.headers),
content=resp.content,
)
flow.response = r
def __init__(self, flow: mitmproxy.http.HTTPFlow, parameters: list):
self.origin_flow = flow.copy()
self.parameters = {}
for val in parameters:
self.parameters[val] = (False, True)
def replay(fake_flow: mitmproxy.http.HTTPFlow):
fake_flow.live = False
fake_flow.intercepted = False
if "view" in ctx.master.addons:
ctx.master.commands.call("view.flows.add", [fake_flow])
ctx.master.commands.call("replay.client", [fake_flow])
def request(flow: mitmproxy.http.HTTPFlow) -> None:
if config.mock_configuration is None:
print(f'Passing request to {str(flow.request.url)}')
return None
else:
pass
matching_mocks = [
mock for mock in config.mock_configuration.mocks
if re.match(mock.path, str(flow.request.url)) and mock.enabled
]
if len(matching_mocks) <= 0:
print(f'Passing request to {str(flow.request.url)}')
return None
else:
pass
mock = matching_mocks[0]
if mock is not None:
if mock.interactive:
print(f'Wating for send confirmation for {str(flow.request.url)}')
input('\x1b[1;30;41mPress Return to continue...\x1b[0m')
else:
pass
if mock.offline:
if mock.status_code is not None:
print(
f'Responding offline to request to {str(flow.request.url)}'
)
print(f'...using status code {mock.status_code}')
response = mitmproxy.http.Response.make(
mock.status_code,
b'',
{},
)
else:
print(
f'Invalid mock, passing request to {str(flow.request.url)}'
)
return None
for k, v in mock.headers.items():
print(f'...using header {k} with {v}')
response.headers[k] = v
if mock.body_path is not None:
print(f'...using body with content of {mock.body_path}')
with open(f'{config.mock_configuration.path}/{mock.body_path}',
'r') as mock_data:
response.content = str.encode(mock_data.read())
elif mock.body is not None:
print(f'...using body with {mock.body}')
response.content = mock.body.encode('utf-8')
else:
pass
flow.response = response
else:
print(f'Passing request to {str(flow.request.url)}')
else:
print(f'Passing request to {str(flow.request.url)}')