def replay_request(
self,
f: http.HTTPFlow,
block: bool=False
) -> http_replay.RequestReplayThread:
"""
Replay a HTTP request to receive a new response from the server.
Args:
f: The flow to replay.
block: If True, this function will wait for the replay to finish.
This causes a deadlock if activated in the main thread.
Returns:
The thread object doing the replay.
Raises:
exceptions.ReplayException, if the flow is in a state
where it is ineligible for replay.
"""
if f.live:
raise exceptions.ReplayException(
"Can't replay live flow."
)
if f.intercepted:
raise exceptions.ReplayException(
"Can't replay intercepted flow."
)
if not f.request:
raise exceptions.ReplayException(
"Can't replay flow with missing request."
)
if f.request.raw_content is None:
raise exceptions.ReplayException(
"Can't replay flow with missing content."
)
f.backup()
f.request.is_replay = True
f.response = None
f.error = None
if f.request.http_version == "HTTP/2.0": # https://github.com/mitmproxy/mitmproxy/issues/2197
f.request.http_version = "HTTP/1.1"
host = f.request.headers.pop(":authority")
f.request.headers.insert(0, "host", host)
rt = http_replay.RequestReplayThread(
self.options,
f,
self.event_queue,
self.should_exit
)
rt.start() # pragma: no cover
if block:
rt.join()
return rt
def request(flow: http.HTTPFlow) -> None:
# pretty_url takes the "Host" header of the request into account, which
# is useful in transparent mode where we usually only have the IP otherwise.
if flow.request.pretty_url == "http://example.com/path":
flow.response = http.HTTPResponse.make(
200, # (optional) status code
b"Hello World", # (optional) content
{"Content-Type": "text/html"} # (optional) headers
)
def request(self, flow: http.HTTPFlow):
"""
使用request事件实现map local
:param flow:
:return:
"""
if "https://stock.xueqiu.com/v5/stock/batch/quote.json?_t=" in flow.request.pretty_url:
with open("D:\\ceshiren\\test_mock\\xueqiu_batch_quote.json", encoding="utf-8") as f:
# 给flow.response属性进行赋值，
# 赋值调用mitmproxy 响应对象的 make方法
# 响应体在make函数里面所需要的数据为str
flow.response = http.HTTPResponse.make(200, # (optional) status code
f.read(), # (optional) content
{"Content-Type": "text/html"} # (optional) headers
)
def replay_request(self,
f: http.HTTPFlow,
block: bool = False) -> http_replay.RequestReplayThread:
"""
Replay a HTTP request to receive a new response from the server.
Args:
f: The flow to replay.
block: If True, this function will wait for the replay to finish.
This causes a deadlock if activated in the main thread.
Returns:
The thread object doing the replay.
Raises:
exceptions.ReplayException, if the flow is in a state
where it is ineligible for replay.
"""
if f.live:
raise exceptions.ReplayException("Can't replay live flow.")
if f.intercepted:
raise exceptions.ReplayException("Can't replay intercepted flow.")
if not f.request:
raise exceptions.ReplayException(
"Can't replay flow with missing request.")
if f.request.raw_content is None:
raise exceptions.ReplayException(
"Can't replay flow with missing content.")
f.backup()
f.request.is_replay = True
f.response = None
f.error = None
if f.request.http_version == "HTTP/2.0": # https://github.com/mitmproxy/mitmproxy/issues/2197
f.request.http_version = "HTTP/1.1"
host = f.request.headers.pop(":authority")
f.request.headers.insert(0, "host", host)
rt = http_replay.RequestReplayThread(self.server.config, f,
self.event_queue,
self.should_exit)
rt.start() # pragma: no cover
if block:
rt.join()
return rt
async def serve(app, flow: http.HTTPFlow):
"""
Serves app on flow.
"""
assert flow.reply
scope = make_scope(flow)
done = asyncio.Event()
received_body = False
sent_response = False
async def receive():
nonlocal received_body
if not received_body:
received_body = True
return {
"type": "http.request",
"body": flow.request.raw_content,
}
else: # pragma: no cover
# We really don't expect this to be called a second time, but what to do?
# We just wait until the request is done before we continue here with sending a disconnect.
await done.wait()
return {"type": "http.disconnect"}
async def send(event):
if event["type"] == "http.response.start":
flow.response = http.Response.make(event["status"], b"",
event.get("headers", []))
flow.response.decode()
elif event["type"] == "http.response.body":
flow.response.content += event.get("body", b"")
if not event.get("more_body", False):
nonlocal sent_response
sent_response = True
else:
raise AssertionError(f"Unexpected event: {event['type']}")
try:
await app(scope, receive, send)
if not sent_response:
raise RuntimeError(f"no response sent.")
except Exception:
ctx.log.error(f"Error in asgi app:\n{traceback.format_exc(limit=-5)}")
flow.response = http.Response.make(500, b"ASGI Error.")
finally:
flow.reply.commit()
done.set()
def request(flow: http.HTTPFlow) -> None:
# pretty_url takes the "Host" header of the request into account, which
# is useful in transparent mode where we usually only have the IP otherwise.
for api in apiconfig:
if api[api_path] in flow.request.pretty_url and api[
mock_response] == True:
print(api[api_path])
mockjson = open(api[response_json_path], "r").read()
print("response: ", mockjson)
flow.response = http.HTTPResponse.make(
400, # (optional) status code
mockjson, # (optional) content
{"Content-Type": "application/json"} # (optional) headers
)
break
def request(self, f: http.HTTPFlow) -> None:
if self.flowmap:
rflow = self.next_flow(f)
if rflow:
assert rflow.response
response = rflow.response.copy()
response.is_replay = True
if ctx.options.server_replay_refresh:
response.refresh()
f.response = response
elif ctx.options.server_replay_kill_extra:
ctx.log.warn(
"server_playback: killed non-replay request {}".format(
f.request.url))
assert f.reply
f.reply.kill()
def request(flow: http.HTTPFlow) -> None:
"""
实现类似与Charles中的map local功能
:param flow:
:return:
"""
# pretty_url takes the "Host" header of the request into account, which
# is useful in transparent mode where we usually only have the IP otherwise.
# 如果请求参数中包含"quote.json"和"x="，则将文件response.json中的内容作为响应，返回给移动端
if "quote.json" in flow.request.pretty_url and "x=" in flow.request.pretty_url:
with open("response.json", encoding="utf-8") as f:
flow.response = http.HTTPResponse.make(
200, # (optional) status code
f.read(), # (optional) content
{"Content-Type": "application/json"} # (optional) headers
)
def request(self, flow: http.HTTPFlow):
if any([filter(flow) for filter in self.flowfilters]):
flow.request.replace(r'result=false', 'result=true')
return
if self.mp4(flow):
ctx.log.error('mp4 url hit......')
ctx.log.error(flow.request.url)
# This example shows how to send a reply from the proxy immediately
# without sending any data to the remote server.
flow.response = http.HTTPResponse.make(
200, # (optional) status code
self.content, # (optional) content
{"Content-Type": "video/mp4"})
return
def request(flow: http.HTTPFlow) -> None:
if flow.request.path.split('?')[0] not in PATHS: return
CV[flow] = threading.Condition()
ts = []
for h in HANDLERS:
t = PVThread(target=h, args=(flow, ))
t.daemon = True
t.start()
with CV[flow]:
CV[flow].notify_all()
with CV[flow]:
if not CV[flow].wait(timeout=1.5):
ctx.log.warn("<!> Wait timeout: request match error!")
flow.response = http.make_error_response(
500, "Wait timeout: request match error!")
def request(self, flow: http.HTTPFlow):
"""
use request event to complete map local
:param flow:
"""
if "https://stock.xueqiu.com/v5/stock/batch/quote.json?_t=" in flow.request.pretty_url:
with open(
"/Users/jun_lei/Documents/Jun/code/python/SnowBallTest/test_mock/test.json",
encoding="utf-8") as f:
# assign value to flow.response attribute
# call mitmproxy response object make function
# the data in make function in response body is str
flow.response = http.HTTPResponse.make(
200, # (optional) status code
f.read(), # (optional) content
{"Content-Type": "text/html"} # (optional) headers
)
def request(flow: http.HTTPFlow) -> None:
# pretty_url takes the "Host" header of the request into account, which
# is useful in transparent mode where we usually only have the IP otherwise.
#当我们的返回的url中包含quote.json,就会执行下面的map local
#雪球的行情返回的url就是quote.json，可以从这方面入手
if "quote.json" in flow.request.pretty_url:
#我们通过charles的mirror拿到quote.json，改造成1.json放到项目里面去吧
#真烦每次我都要UTF-8，歧视windows
with open("1.json", encoding="UTF-8") as f:
#这里一个字典
flow.response = http.HTTPResponse.make(
200, # (optional) status code
f.read(), #直接读取我们的json文件，套入到响应，等于响应就是这个1.json
#由于传的是json，所以content-type改成json的
{"Content-Type": "application/json"} # (optional) headers
)
def ws_testdata(tctx):
tctx.server.address = ("example.com", 80)
tctx.server.state = ConnectionState.OPEN
flow = HTTPFlow(tctx.client, tctx.server)
flow.request = Request.make("GET",
"http://example.com/",
headers={
"Connection": "upgrade",
"Upgrade": "websocket",
"Sec-WebSocket-Version": "13",
})
flow.response = Response.make(101,
headers={
"Connection": "upgrade",
"Upgrade": "websocket",
})
return tctx, Playbook(websocket.WebsocketLayer(tctx, flow))
def request(self, flow: http.HTTPFlow):
if flow.request.path.find("/gate.php") != -1: # make sure that remote host is not localhost
# Log request
self.simLogger.warning("---> Matching Request")
httpReqStr = utils.createHttpRequestLogMsg(flow, 0)
self.simLogger.warning(httpReqStr)
helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpReqStr, flow, 0, helpers.HttpPacketType.Request)
# Create response
config = "IS_G_PWDS: 1\r\n" + "IS_G_DOUBLE: 1\r\n" + "IS_G_BROWSERS: 1\r\n" + "IS_G_COINS: 1\r\n" + "IS_G_SKYPE: 1\r\n" + "IS_G_STEAM: 1\r\n" + "IS_G_DESKTOP: 1\r\n" + "G_DESKTOP_EXTS: txt,doc\r\n" + "G_DESKTOP_MAXSIZE: 200\r\n"
flow.response = http.HTTPResponse.make(200, config, {"Content-Type": "text/plain; charset=utf-8"})
# Log response
self.simLogger.warning("---> Own Response")
httpResStr = utils.createHttpResponseLogMsg(flow,0)
self.simLogger.warning(httpResStr)
helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpResStr, flow, 0, helpers.HttpPacketType.Response)
def request(self, flow: http.HTTPFlow):
print('[Proxy] HTTP request')
# The default homepage at http://pntest
if flow.request.host == 'pntest':
flow.response = http.Response.make(200, self.pntest_homepage_html,
{"content-type": "text/html"})
request_state = flow.request.get_state()
request_state['flow_uuid'] = flow.id
request_state['type'] = 'request'
request_state['client_id'] = self.client_id
request_state['intercepted'] = self.__should_intercept_request(flow)
self.__send_message(request_state)
if request_state['intercepted']:
self.intercept_flow(flow)
def request(self, flow: http.HTTPFlow):
if flow.request.path.find("Libs.zip") != -1:
# Log request
self.simLogger.warning("---> Matching Request")
httpReqStr = utils.createHttpRequestLogMsg(flow, 0)
self.simLogger.warning(httpReqStr)
helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpReqStr, flow, 0, helpers.HttpPacketType.Request)
fLibs = open("AcridRain_Parts/Libs.zip", "rb") # exactly same
responseBytes = fLibs.read()
flow.response = http.HTTPResponse.make(200, responseBytes, {"Content-Type": "application/zip"})
fLibs.close()
# Log response
self.simLogger.warning("---> Own Response")
httpResStr = utils.createHttpResponseLogMsg(flow, 0)
self.simLogger.warning(httpResStr)
helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpResStr, flow, 0, helpers.HttpPacketType.Response)
def block_playtime(flow: HTTPFlow):
if config.block_playtime and flow.request.path.startswith('/library/api/public/playtime/'):
original_playtime = json.loads(flow.request.text)
flow.request.text = '{}' # Just in case
correlation_id = flow.request.headers.get('X-Epic-Correlation-ID')
if m := re.match(r"UE4-(\w+)", correlation_id):
device_id = m.group(1)
else:
device_id = '123456789abcdef01234567890abcdef'
flow.response = HTTPResponse.make(204)
flow.response.headers.add('x-epic-device-id', device_id)
flow.response.headers.add('x-epic-correlation-id', correlation_id)
flow.response.headers.add('x-envoy-upstream-service-time', '10') # ?
flow.response.headers.add('server', 'istio-envoy')
log.info('Blocked playtime request from Epic Games')
log.debug(f'\n{json.dumps(original_playtime, indent=4)}')
def request(self, flow: http.HTTPFlow):
method, url = flow.request.method, flow.request.url
mock_datas = db.session.query(MockData).filter_by(method=method, status=1).all()
match_data = self.get_matched_data(url, mock_datas)
if match_data:
logger.info('%s%6s %s', '▶', method, flow.request.path)
content = json.dumps(self.mock(match_data.response), ensure_ascii=False) if match_data.response else ''
content_type = {"Content-Type": match_data.content_type} if match_data.content_type else {}
headers = self.mock(match_data.headers) if match_data.headers else {}
logger.debug('Mock data: %s', content)
flow.response = http.HTTPResponse.make(
match_data.code, # (optional) status code
content, # (optional) content
{'Access-Control-Allow-Origin': '*', **content_type, **headers}
)
else:
logger.info('%-2s%6s %s', '•', method, flow.request.path[:200] + (flow.request.path[200:] and '...'))
def request(flow: http.HTTPFlow) -> None:
routers = utils.readFile(ROUTER_FILE)
url = flow.request.url
ctx.log.info(url)
if routers is not None:
for patternURL, jsonfilename in routers.items():
if re.match(patternURL, url) is not None:
jsonfile = DATA_DIR + str(jsonfilename) + '.json'
data = utils.readFile(jsonfile)
if data is not None:
status = int(data['status'])
try:
content = json.dumps(data['content'])
except:
content = ''
header = data['header']
flow.response = http.HTTPResponse.make(
status, content, header)
def request(self, flow: http.HTTPFlow):
if flow.request.path.find("config.php") != -1 or flow.request.pretty_host == "a-n-y.online" or flow.request.pretty_host == "jelouslaodnn.org" or flow.request.pretty_host == "seeyouonlineservice.com":
# Log request
self.simLogger.warning("---> Matching Request")
httpReqStr = utils.createHttpRequestLogMsg(flow, 0)
self.simLogger.warning(httpReqStr)
helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpReqStr, flow, 0, helpers.HttpPacketType.Request)
if(flow.request.path.find("config.php") != -1):
# Create response
fConfig = open("Kpot_Parts/config_kpot.bin", "rb") # exactly same
responseBytes = fConfig.read()
fConfig.close()
flow.response = http.HTTPResponse.make(200, responseBytes, {"Content-Type": "text/plain; charset=utf-8"})
# Log response
self.simLogger.warning("---> Own Response")
httpResStr = utils.createHttpResponseLogMsg(flow,0)
self.simLogger.warning(httpResStr)
helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpResStr, flow, 0, helpers.HttpPacketType.Response)
def request(flow: http.HTTPFlow) -> None:
ctx.log.debug("Request {}: {}".format(flow.request.path, flow.request))
config = resolve_config(flow, "request")
if config is None:
return
required_headers = config.get("headers")
if required_headers and not content_matches(None, required_headers,
dict(flow.request.headers)):
return
ctx.log.debug("Match request {}: {}".format(flow.request.path, config))
save = config.get("save", mock_config.get("save"))
if save:
save_flow(save, flow, "request")
modify = config.get("modify")
if modify:
ctx.log.debug("Modify request: {} -> {}".format(flow.request, modify))
flow.request.scheme = modify.get("scheme", flow.request.scheme)
flow.request.host = modify.get("host", flow.request.host)
flow.request.path = modify.get("path", flow.request.path)
flow.request.method = modify.get("method", flow.request.method)
query_modifier = modify.get("query")
if query_modifier:
query = flow.request.query or {}
if isinstance(query_modifier, str) or isinstance(
query_modifier, list):
flow.request.query = content_as_object(
modify_content(query_modifier, dict(query)))
else:
flow.request.query = {**query, **query_modifier}
flow.request.headers.update(modify.get("headers", {}))
modifier = modify.get("content")
if modifier is not None:
content = flow.request.text or ""
flow.request.text = content_as_str(
modify_content(modifier, content))
response = config.get("respond")
if response:
flow.response = make_response(response, 200, "", {})
ctx.log.debug("Mock {}".format(flow.request.path))
def request(self, flow: HTTPFlow):
if flow.request.host in self.ServersList and flow.request.path.startswith(
"/quest/battleStart"):
req = json.loads(flow.request.get_text())
if (req["stageId"] != "main_08-16"):
return
self.info("Receive JT8-2 battle start request")
fakeData = {
"apFailReturn": 20,
"battleId": "6c86c7c0-3373-11eb-9784-0d36b8275660",
"isApProtect": 0,
"notifyPowerScoreNotEnoughIfFailed": False,
"playerDataDelta": {
"deleted": {},
"modified": {
# "dungeon": {
# "stages": {
# "main_07-01": {
# "completeTimes": 1,
# "hasBattleReplay": 1,
# "noCostCnt": 0,
# "practiceTimes": 0,
# "stageId": "main_07-01",
# "startTimes": 2,
# "state": 3
# }
# }
# }
}
},
"result": 0
}
flow.response = HTTPResponse.make(
200, json.dumps(fakeData),
{"Content-Type": "application/json; charset=utf-8"})
self.info("complete")
def response(flow: http.HTTPFlow) -> None:
ctx.log.debug("Response {}: {}".format(flow.request.path, flow.response))
config = resolve_config(flow, "response")
if config is None:
return
required_headers = config.get("headers")
if required_headers:
headers = {**dict(flow.request.headers), **dict(flow.response.headers)}
if not content_matches(None, required_headers, headers):
return
ctx.log.debug("Match response {}: {}".format(flow.request.path, config))
save = config.get("save", mock_config.get("save"))
if save:
save_flow(save, flow, "response")
replace = config.get("replace")
if replace:
response = replace.get("response", replace)
if response:
flow.response = make_response(response, flow.response.status_code,
flow.response.content,
flow.response.headers)
ctx.log.debug("Replace response {}: {}".format(
flow.request.path, flow.response))
modify = config.get("modify", [])
if isinstance(modify, dict) or isinstance(modify, str):
modify = [modify]
global_modify = mock_config.get("response", {}).get("*", {}).get("modify")
if global_modify:
if isinstance(global_modify, dict) or isinstance(global_modify, str):
global_modify = [global_modify]
modify = global_modify + modify
if modify:
flow.response.text = content_as_str(
modify_content(modify, flow.response.text))
ctx.log.debug("Modify response {}: {}".format(flow.request.path,
modify))
def response(flow: http.HTTPFlow):
# todo 修改响应的状态码
print("所有状态码%s" % str(flow.response.status_code))
# if '/service/settings/v2/' in flow.request.path:
# flow.response = http.HTTPResponse.make(404)
# ctx.log.error("响应状态码 %s" % str(flow.response.status_code))
# todo rewrite功能修改字段 忽略非360 搜索地址
if flow.request.host != "www.so.com":
return
# 将响应text中所有“搜索”替换为“请使用谷歌”
text = flow.response.get_text()
text = text.replace("搜索", "请使用谷歌")
flow.response.set_text(text)
flow.response.text = json.dumps(text)
# todo 实现Charles中的MapRemote功能
# 加上过滤条件
if "quote.json" in flow.request.pretty_url and "x=" in flow.request.pretty_url:
# 把响应数据转化成python对象，保存到data中
data = json.loads(flow.response.content)
# 对第一个股票保持原样
data['data']['items'][0] = data['data']['items'][0]
# 对第二个股票名字加长一倍
data['data']['items'][1]['quote'][
'name'] = data['data']['items'][1]['quote']['name'] * 2
# 对第三个股票名字变成空
data['data']['items'][2]['quote']['name'] = None
# 把修改后的内容赋值给 response 原始数据格式
flow.response.text = json.dumps(data)
# todo 实现Charles中的MapLocal功能
# 修改判断条件
if "quote.json" in flow.request.pretty_url and "x=" in flow.request.pretty_url:
# 打开保存在本地的数据文件
with open(
"/Users/chenshifeng/MyCode/PythonCode/SFDSZL/interface/quote.json"
) as f:
# 创造一个 response
flow.response = http.HTTPResponse.make(
200, # (optional) status code
# 读取文件中数据作为返回内容
f.read(),
# 指定返回数据的类型
{"Content-Type": "application/json"} # (optional) headers
)
# todo (MapLocal)修改响应的消息体-通过读取json文件的字符串返给客户端
if flow.request.url.startswith(
"https://xxx.x.xxx.com.cn/activityInfo/getPrizeInfo=="):
# 读取文件，在当前文件路径下执行脚本，否则需要写文件的绝对路径；不然会找不到该json文件
with open('getStatus.json', 'rb') as f:
# 从json文件中读取数据成python对象
res = json.load(f)
# 将读取的python对象转成json字符串发送给客户端
flow.response.set_text(json.dumps(res))
ctx.log.info("modify order status")
# todo 方法2 (rewrite)字段 修改响应的消息体-直接修改响应字段
if flow.request.url.startswith(
"https://activity.x.xxx.com.cn/activityPlugDrawInfo/getPrizeInfo=="
):
# 获取响应的json字符串，转成python对象进行解析和修改
response = json.loads(flow.response.get_text())
response['limitCount'] = 1
# 修改完成后，奖python对象转成json字符串，set进请求的响应体重发送给客户端
flow.response.set_text()
ctx.log.info('modify limitCount')
def request(flow: http.HTTPFlow):
# 发起请求，判断 url 是不是预期值
if "events" in flow.request.pretty_url:
# 创造一个 response
flow.response = http.HTTPResponse.make(
200, # (optional) status code
event, # (optional) content
{"Content-Type": "application/json"} # (optional) headers
)
# 发起请求，判断 url 是不是预期值
if "eventtypesdistr" in flow.request.pretty_url:
# 打开一个保存在本地的文件
with open("data/eventtypedistr.json") as f:
# 创造一个 response
flow.response = http.HTTPResponse.make(
200, # (optional) status code
f.read(), # (optional) content
{"Content-Type": "application/json"} # (optional) headers
)
# 发起请求，判断 url 是不是预期值
if "traffic" in flow.request.pretty_url:
# 打开一个保存在本地的文件
with open("data/traffic.json") as f:
# 创造一个 response
flow.response = http.HTTPResponse.make(
200, # (optional) status code
f.read(), # (optional) content
{"Content-Type": "application/json"} # (optional) headers
)
# 发起请求，判断 url 是不是预期值
if "srctopn" in flow.request.pretty_url:
# 打开一个保存在本地的文件
with open("data/srctopn.json") as f:
# 创造一个 response
flow.response = http.HTTPResponse.make(
200, # (optional) status code
f.read(), # (optional) content
{"Content-Type": "application/json"} # (optional) headers
)
# 发起请求，判断 url 是不是预期值
if "/topn/?chart=bar&unit=bps&sortby=In" in flow.request.pretty_url:
# 打开一个保存在本地的文件
with open("data/topn.json") as f:
# 创造一个 response
flow.response = http.HTTPResponse.make(
200, # (optional) status code
f.read(), # (optional) content
{"Content-Type": "application/json"} # (optional) headers
)
# 发起请求，判断 url 是不是预期值
if "eventdistr" in flow.request.pretty_url:
# 打开一个保存在本地的文件
with open("data/eventdistr.json") as f:
# 创造一个 response
flow.response = http.HTTPResponse.make(
200, # (optional) status code
f.read(), # (optional) content
{"Content-Type": "application/json"} # (optional) headers
)
# 发起请求，判断 url 是不是预期值
if "srctopngeodist" in flow.request.pretty_url:
# 打开一个保存在本地的文件
with open("data/srctopngeodist.json") as f:
# 创造一个 response
flow.response = http.HTTPResponse.make(
200, # (optional) status code
f.read(), # (optional) content
{"Content-Type": "application/json"} # (optional) headers
)
# 发起请求，判断 url 是不是预期值
if "abnormaltopn" in flow.request.pretty_url:
# 打开一个保存在本地的文件
with open("data/adnormaltopn.json") as f:
# 创造一个 response
flow.response = http.HTTPResponse.make(
200, # (optional) status code
f.read(), # (optional) content
{"Content-Type": "application/json"} # (optional) headers
)
def response(flow: http.HTTPFlow) -> None:
if random.randint(1, 100) < percentage:
ctx.log.warn('>>> Down ' + flow.request.url)
flow.response = http.HTTPResponse.make(503, '', {})
def request(self, flow: http.HTTPFlow):
"""Handle HTTP Request."""
js_tool = False
full_url = flow.request.pretty_url.split("?")[0]
if "?JSTool=" in flow.request.pretty_url:
full_url += "?JSTool"
js_tool = True
hashed_url = hashlib.sha256(full_url.encode()).hexdigest()
cursor = self.cnx.cursor()
cursor.execute(
"SELECT `file_name` FROM resource_location "
"WHERE `hashed_url` LIKE '%s'" %
(hashed_url)
)
result = cursor.fetchone()
if result:
# found in db
file_name = str(result[0])
file_path = self.path + "/cache/" + file_name
if js_tool:
# headers
with open(file_path + '.h', 'rb') as temp_file:
temp_headers = pickle.load(temp_file)
headers_dict = {}
for attr in list(temp_headers):
headers_dict[attr] = temp_headers[attr]
# make simplification
with open(file_path + '.c', 'rb') as temp_file:
temp_content = pickle.load(temp_file)
html = str(BeautifulSoup(temp_content, 'html.parser'))
active = flow.request.pretty_url.split(
"?JSTool=")[-1].split("_")
active = active[1:]
for num in active:
index = html.find("<!--script" + str(num) + "\n")
html = html.replace(
"<!--script" + str(num) + "\n", "<script")
html = html[0:index] + \
html[index:].replace("-->\n", "</script>", 1)
# return response
flow.response = http.HTTPResponse.make(
200, # (optional) status code
html, # (optional) content
headers_dict)
print("*" * 30)
print(full_url, "served from cache file", file_name)
else:
# delete existing entries
cursor.execute(
"DELETE FROM resource_location "
"WHERE `hashed_url` LIKE '%s'" %
(hashed_url)
)
try:
os.remove(file_path + '.h')
os.remove(file_path + '.c')
except FileNotFoundError:
pass
elif js_tool:
full_url = flow.request.pretty_url.split("?")[0]
hashed_url = hashlib.sha256(full_url.encode()).hexdigest()
cursor.execute(
"SELECT `file_name` FROM resource_location "
"WHERE `hashed_url` LIKE '%s'" %
(hashed_url)
)
result = cursor.fetchone()
if result:
# Add version with scripts removed to cache
file_name = str(result[0])
file_path = self.path + "/cache/" + file_name
with open(file_path + '.h', 'rb') as temp_file:
temp_headers = pickle.load(temp_file)
headers_dict = {}
for attr in list(temp_headers):
headers_dict[attr] = temp_headers[attr]
with open(file_path + '.c', 'rb') as temp_file:
temp_content = pickle.load(temp_file)
html = str(BeautifulSoup(temp_content, 'html.parser'))
# Remove comments
while "<!--" in html:
start_index = html.find("<!--")
end_index = html.find("-->") + 3
html = html.replace(html[start_index:end_index], "")
cnt = 1
while "<script" in html:
html = html.replace(
"<script", "\n<!--script" + str(cnt) + "\n", 1)
html = html.replace("</script>", "\n-->\n", 1)
print("cnt =", cnt)
cnt += 1
# return response
flow.response = http.HTTPResponse.make(
200, # (optional) status code
html, # (optional) content
headers_dict)
# add to database
full_url += "?JSTool"
hashed_url = hashlib.sha256(full_url.encode()).hexdigest()
file_name = self.random_filename()
file_path = self.path + "/cache/" + file_name
cursor.execute(
"INSERT INTO resource_location "
"(`hashed_url`, `full_url`, `file_name`, datetime) "
"VALUES ('%s', '%s', '%s', NOW())" %
(hashed_url, full_url, file_name)
)
self.cnx.commit()
# write to file
with open(file_path + '.h', 'wb') as temp_file:
pickle.dump(flow.response.headers, temp_file)
with open(file_path + '.c', 'wb') as temp_file:
pickle.dump(flow.response.content, temp_file)
cursor.close()
def request(self, flow: http.HTTPFlow):
if flow.request.path == "/cspscannerreport":
flow.response = http.HTTPResponse.make(200)
def request(flow: http.HTTPFlow):
if flow.request.pretty_url == "https://www.baidu.com/":
flow.response = http.HTTPResponse.make(200, b"Hello Word",
{"Content-Type": "text/html"})
def request(flow: http.HTTPFlow) -> None:
if (flow.request.pretty_host == "yatsun.me"):
flow.response = http.HTTPResponse.make(
200, '<html><head></head><body><h1>Test</h1></body></html>',
{"Content-Type": "text/html"})
def request(flow: http.HTTPFlow):
if "baidu.com" in flow.request.pretty_url:
# if flow.request.pretty_url == "http://example.com/path":
flow.response = http.HTTPResponse.make(200, b"Hello World",
{"Content-Type": "text/html"})
def reply_from_proxy(flow: HTTPFlow):
flow.response = Response.make(418)