def replay_request( self, f: http.HTTPFlow, block: bool=False ) -> http_replay.RequestReplayThread: """ Replay a HTTP request to receive a new response from the server. Args: f: The flow to replay. block: If True, this function will wait for the replay to finish. This causes a deadlock if activated in the main thread. Returns: The thread object doing the replay. Raises: exceptions.ReplayException, if the flow is in a state where it is ineligible for replay. """ if f.live: raise exceptions.ReplayException( "Can't replay live flow." ) if f.intercepted: raise exceptions.ReplayException( "Can't replay intercepted flow." ) if not f.request: raise exceptions.ReplayException( "Can't replay flow with missing request." ) if f.request.raw_content is None: raise exceptions.ReplayException( "Can't replay flow with missing content." ) f.backup() f.request.is_replay = True f.response = None f.error = None if f.request.http_version == "HTTP/2.0": # https://github.com/mitmproxy/mitmproxy/issues/2197 f.request.http_version = "HTTP/1.1" host = f.request.headers.pop(":authority") f.request.headers.insert(0, "host", host) rt = http_replay.RequestReplayThread( self.options, f, self.event_queue, self.should_exit ) rt.start() # pragma: no cover if block: rt.join() return rt
def request(flow: http.HTTPFlow) -> None: # pretty_url takes the "Host" header of the request into account, which # is useful in transparent mode where we usually only have the IP otherwise. if flow.request.pretty_url == "http://example.com/path": flow.response = http.HTTPResponse.make( 200, # (optional) status code b"Hello World", # (optional) content {"Content-Type": "text/html"} # (optional) headers )
def request(self, flow: http.HTTPFlow): """ 使用request事件实现map local :param flow: :return: """ if "https://stock.xueqiu.com/v5/stock/batch/quote.json?_t=" in flow.request.pretty_url: with open("D:\\ceshiren\\test_mock\\xueqiu_batch_quote.json", encoding="utf-8") as f: # 给flow.response属性进行赋值, # 赋值调用mitmproxy 响应对象的 make方法 # 响应体在make函数里面所需要的数据为str flow.response = http.HTTPResponse.make(200, # (optional) status code f.read(), # (optional) content {"Content-Type": "text/html"} # (optional) headers )
def replay_request(self, f: http.HTTPFlow, block: bool = False) -> http_replay.RequestReplayThread: """ Replay a HTTP request to receive a new response from the server. Args: f: The flow to replay. block: If True, this function will wait for the replay to finish. This causes a deadlock if activated in the main thread. Returns: The thread object doing the replay. Raises: exceptions.ReplayException, if the flow is in a state where it is ineligible for replay. """ if f.live: raise exceptions.ReplayException("Can't replay live flow.") if f.intercepted: raise exceptions.ReplayException("Can't replay intercepted flow.") if not f.request: raise exceptions.ReplayException( "Can't replay flow with missing request.") if f.request.raw_content is None: raise exceptions.ReplayException( "Can't replay flow with missing content.") f.backup() f.request.is_replay = True f.response = None f.error = None if f.request.http_version == "HTTP/2.0": # https://github.com/mitmproxy/mitmproxy/issues/2197 f.request.http_version = "HTTP/1.1" host = f.request.headers.pop(":authority") f.request.headers.insert(0, "host", host) rt = http_replay.RequestReplayThread(self.server.config, f, self.event_queue, self.should_exit) rt.start() # pragma: no cover if block: rt.join() return rt
async def serve(app, flow: http.HTTPFlow): """ Serves app on flow. """ assert flow.reply scope = make_scope(flow) done = asyncio.Event() received_body = False sent_response = False async def receive(): nonlocal received_body if not received_body: received_body = True return { "type": "http.request", "body": flow.request.raw_content, } else: # pragma: no cover # We really don't expect this to be called a second time, but what to do? # We just wait until the request is done before we continue here with sending a disconnect. await done.wait() return {"type": "http.disconnect"} async def send(event): if event["type"] == "http.response.start": flow.response = http.Response.make(event["status"], b"", event.get("headers", [])) flow.response.decode() elif event["type"] == "http.response.body": flow.response.content += event.get("body", b"") if not event.get("more_body", False): nonlocal sent_response sent_response = True else: raise AssertionError(f"Unexpected event: {event['type']}") try: await app(scope, receive, send) if not sent_response: raise RuntimeError(f"no response sent.") except Exception: ctx.log.error(f"Error in asgi app:\n{traceback.format_exc(limit=-5)}") flow.response = http.Response.make(500, b"ASGI Error.") finally: flow.reply.commit() done.set()
def request(flow: http.HTTPFlow) -> None: # pretty_url takes the "Host" header of the request into account, which # is useful in transparent mode where we usually only have the IP otherwise. for api in apiconfig: if api[api_path] in flow.request.pretty_url and api[ mock_response] == True: print(api[api_path]) mockjson = open(api[response_json_path], "r").read() print("response: ", mockjson) flow.response = http.HTTPResponse.make( 400, # (optional) status code mockjson, # (optional) content {"Content-Type": "application/json"} # (optional) headers ) break
def request(self, f: http.HTTPFlow) -> None: if self.flowmap: rflow = self.next_flow(f) if rflow: assert rflow.response response = rflow.response.copy() response.is_replay = True if ctx.options.server_replay_refresh: response.refresh() f.response = response elif ctx.options.server_replay_kill_extra: ctx.log.warn( "server_playback: killed non-replay request {}".format( f.request.url)) assert f.reply f.reply.kill()
def request(flow: http.HTTPFlow) -> None: """ 实现类似与Charles中的map local功能 :param flow: :return: """ # pretty_url takes the "Host" header of the request into account, which # is useful in transparent mode where we usually only have the IP otherwise. # 如果请求参数中包含"quote.json"和"x=",则将文件response.json中的内容作为响应,返回给移动端 if "quote.json" in flow.request.pretty_url and "x=" in flow.request.pretty_url: with open("response.json", encoding="utf-8") as f: flow.response = http.HTTPResponse.make( 200, # (optional) status code f.read(), # (optional) content {"Content-Type": "application/json"} # (optional) headers )
def request(self, flow: http.HTTPFlow): if any([filter(flow) for filter in self.flowfilters]): flow.request.replace(r'result=false', 'result=true') return if self.mp4(flow): ctx.log.error('mp4 url hit......') ctx.log.error(flow.request.url) # This example shows how to send a reply from the proxy immediately # without sending any data to the remote server. flow.response = http.HTTPResponse.make( 200, # (optional) status code self.content, # (optional) content {"Content-Type": "video/mp4"}) return
def request(flow: http.HTTPFlow) -> None: if flow.request.path.split('?')[0] not in PATHS: return CV[flow] = threading.Condition() ts = [] for h in HANDLERS: t = PVThread(target=h, args=(flow, )) t.daemon = True t.start() with CV[flow]: CV[flow].notify_all() with CV[flow]: if not CV[flow].wait(timeout=1.5): ctx.log.warn("<!> Wait timeout: request match error!") flow.response = http.make_error_response( 500, "Wait timeout: request match error!")
def request(self, flow: http.HTTPFlow): """ use request event to complete map local :param flow: """ if "https://stock.xueqiu.com/v5/stock/batch/quote.json?_t=" in flow.request.pretty_url: with open( "/Users/jun_lei/Documents/Jun/code/python/SnowBallTest/test_mock/test.json", encoding="utf-8") as f: # assign value to flow.response attribute # call mitmproxy response object make function # the data in make function in response body is str flow.response = http.HTTPResponse.make( 200, # (optional) status code f.read(), # (optional) content {"Content-Type": "text/html"} # (optional) headers )
def request(flow: http.HTTPFlow) -> None: # pretty_url takes the "Host" header of the request into account, which # is useful in transparent mode where we usually only have the IP otherwise. #当我们的返回的url中包含quote.json,就会执行下面的map local #雪球的行情返回的url就是quote.json,可以从这方面入手 if "quote.json" in flow.request.pretty_url: #我们通过charles的mirror拿到quote.json,改造成1.json放到项目里面去吧 #真烦每次我都要UTF-8,歧视windows with open("1.json", encoding="UTF-8") as f: #这里一个字典 flow.response = http.HTTPResponse.make( 200, # (optional) status code f.read(), #直接读取我们的json文件,套入到响应,等于响应就是这个1.json #由于传的是json,所以content-type改成json的 {"Content-Type": "application/json"} # (optional) headers )
def ws_testdata(tctx): tctx.server.address = ("example.com", 80) tctx.server.state = ConnectionState.OPEN flow = HTTPFlow(tctx.client, tctx.server) flow.request = Request.make("GET", "http://example.com/", headers={ "Connection": "upgrade", "Upgrade": "websocket", "Sec-WebSocket-Version": "13", }) flow.response = Response.make(101, headers={ "Connection": "upgrade", "Upgrade": "websocket", }) return tctx, Playbook(websocket.WebsocketLayer(tctx, flow))
def request(self, flow: http.HTTPFlow): if flow.request.path.find("/gate.php") != -1: # make sure that remote host is not localhost # Log request self.simLogger.warning("---> Matching Request") httpReqStr = utils.createHttpRequestLogMsg(flow, 0) self.simLogger.warning(httpReqStr) helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpReqStr, flow, 0, helpers.HttpPacketType.Request) # Create response config = "IS_G_PWDS: 1\r\n" + "IS_G_DOUBLE: 1\r\n" + "IS_G_BROWSERS: 1\r\n" + "IS_G_COINS: 1\r\n" + "IS_G_SKYPE: 1\r\n" + "IS_G_STEAM: 1\r\n" + "IS_G_DESKTOP: 1\r\n" + "G_DESKTOP_EXTS: txt,doc\r\n" + "G_DESKTOP_MAXSIZE: 200\r\n" flow.response = http.HTTPResponse.make(200, config, {"Content-Type": "text/plain; charset=utf-8"}) # Log response self.simLogger.warning("---> Own Response") httpResStr = utils.createHttpResponseLogMsg(flow,0) self.simLogger.warning(httpResStr) helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpResStr, flow, 0, helpers.HttpPacketType.Response)
def request(self, flow: http.HTTPFlow): print('[Proxy] HTTP request') # The default homepage at http://pntest if flow.request.host == 'pntest': flow.response = http.Response.make(200, self.pntest_homepage_html, {"content-type": "text/html"}) request_state = flow.request.get_state() request_state['flow_uuid'] = flow.id request_state['type'] = 'request' request_state['client_id'] = self.client_id request_state['intercepted'] = self.__should_intercept_request(flow) self.__send_message(request_state) if request_state['intercepted']: self.intercept_flow(flow)
def request(self, flow: http.HTTPFlow): if flow.request.path.find("Libs.zip") != -1: # Log request self.simLogger.warning("---> Matching Request") httpReqStr = utils.createHttpRequestLogMsg(flow, 0) self.simLogger.warning(httpReqStr) helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpReqStr, flow, 0, helpers.HttpPacketType.Request) fLibs = open("AcridRain_Parts/Libs.zip", "rb") # exactly same responseBytes = fLibs.read() flow.response = http.HTTPResponse.make(200, responseBytes, {"Content-Type": "application/zip"}) fLibs.close() # Log response self.simLogger.warning("---> Own Response") httpResStr = utils.createHttpResponseLogMsg(flow, 0) self.simLogger.warning(httpResStr) helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpResStr, flow, 0, helpers.HttpPacketType.Response)
def block_playtime(flow: HTTPFlow): if config.block_playtime and flow.request.path.startswith('/library/api/public/playtime/'): original_playtime = json.loads(flow.request.text) flow.request.text = '{}' # Just in case correlation_id = flow.request.headers.get('X-Epic-Correlation-ID') if m := re.match(r"UE4-(\w+)", correlation_id): device_id = m.group(1) else: device_id = '123456789abcdef01234567890abcdef' flow.response = HTTPResponse.make(204) flow.response.headers.add('x-epic-device-id', device_id) flow.response.headers.add('x-epic-correlation-id', correlation_id) flow.response.headers.add('x-envoy-upstream-service-time', '10') # ? flow.response.headers.add('server', 'istio-envoy') log.info('Blocked playtime request from Epic Games') log.debug(f'\n{json.dumps(original_playtime, indent=4)}')
def request(self, flow: http.HTTPFlow): method, url = flow.request.method, flow.request.url mock_datas = db.session.query(MockData).filter_by(method=method, status=1).all() match_data = self.get_matched_data(url, mock_datas) if match_data: logger.info('%s%6s %s', '▶', method, flow.request.path) content = json.dumps(self.mock(match_data.response), ensure_ascii=False) if match_data.response else '' content_type = {"Content-Type": match_data.content_type} if match_data.content_type else {} headers = self.mock(match_data.headers) if match_data.headers else {} logger.debug('Mock data: %s', content) flow.response = http.HTTPResponse.make( match_data.code, # (optional) status code content, # (optional) content {'Access-Control-Allow-Origin': '*', **content_type, **headers} ) else: logger.info('%-2s%6s %s', '•', method, flow.request.path[:200] + (flow.request.path[200:] and '...'))
def request(flow: http.HTTPFlow) -> None: routers = utils.readFile(ROUTER_FILE) url = flow.request.url ctx.log.info(url) if routers is not None: for patternURL, jsonfilename in routers.items(): if re.match(patternURL, url) is not None: jsonfile = DATA_DIR + str(jsonfilename) + '.json' data = utils.readFile(jsonfile) if data is not None: status = int(data['status']) try: content = json.dumps(data['content']) except: content = '' header = data['header'] flow.response = http.HTTPResponse.make( status, content, header)
def request(self, flow: http.HTTPFlow): if flow.request.path.find("config.php") != -1 or flow.request.pretty_host == "a-n-y.online" or flow.request.pretty_host == "jelouslaodnn.org" or flow.request.pretty_host == "seeyouonlineservice.com": # Log request self.simLogger.warning("---> Matching Request") httpReqStr = utils.createHttpRequestLogMsg(flow, 0) self.simLogger.warning(httpReqStr) helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpReqStr, flow, 0, helpers.HttpPacketType.Request) if(flow.request.path.find("config.php") != -1): # Create response fConfig = open("Kpot_Parts/config_kpot.bin", "rb") # exactly same responseBytes = fConfig.read() fConfig.close() flow.response = http.HTTPResponse.make(200, responseBytes, {"Content-Type": "text/plain; charset=utf-8"}) # Log response self.simLogger.warning("---> Own Response") httpResStr = utils.createHttpResponseLogMsg(flow,0) self.simLogger.warning(httpResStr) helpers.saveHttpInfos(self.sampleLogDir + "sim/", httpResStr, flow, 0, helpers.HttpPacketType.Response)
def request(flow: http.HTTPFlow) -> None: ctx.log.debug("Request {}: {}".format(flow.request.path, flow.request)) config = resolve_config(flow, "request") if config is None: return required_headers = config.get("headers") if required_headers and not content_matches(None, required_headers, dict(flow.request.headers)): return ctx.log.debug("Match request {}: {}".format(flow.request.path, config)) save = config.get("save", mock_config.get("save")) if save: save_flow(save, flow, "request") modify = config.get("modify") if modify: ctx.log.debug("Modify request: {} -> {}".format(flow.request, modify)) flow.request.scheme = modify.get("scheme", flow.request.scheme) flow.request.host = modify.get("host", flow.request.host) flow.request.path = modify.get("path", flow.request.path) flow.request.method = modify.get("method", flow.request.method) query_modifier = modify.get("query") if query_modifier: query = flow.request.query or {} if isinstance(query_modifier, str) or isinstance( query_modifier, list): flow.request.query = content_as_object( modify_content(query_modifier, dict(query))) else: flow.request.query = {**query, **query_modifier} flow.request.headers.update(modify.get("headers", {})) modifier = modify.get("content") if modifier is not None: content = flow.request.text or "" flow.request.text = content_as_str( modify_content(modifier, content)) response = config.get("respond") if response: flow.response = make_response(response, 200, "", {}) ctx.log.debug("Mock {}".format(flow.request.path))
def request(self, flow: HTTPFlow): if flow.request.host in self.ServersList and flow.request.path.startswith( "/quest/battleStart"): req = json.loads(flow.request.get_text()) if (req["stageId"] != "main_08-16"): return self.info("Receive JT8-2 battle start request") fakeData = { "apFailReturn": 20, "battleId": "6c86c7c0-3373-11eb-9784-0d36b8275660", "isApProtect": 0, "notifyPowerScoreNotEnoughIfFailed": False, "playerDataDelta": { "deleted": {}, "modified": { # "dungeon": { # "stages": { # "main_07-01": { # "completeTimes": 1, # "hasBattleReplay": 1, # "noCostCnt": 0, # "practiceTimes": 0, # "stageId": "main_07-01", # "startTimes": 2, # "state": 3 # } # } # } } }, "result": 0 } flow.response = HTTPResponse.make( 200, json.dumps(fakeData), {"Content-Type": "application/json; charset=utf-8"}) self.info("complete")
def response(flow: http.HTTPFlow) -> None: ctx.log.debug("Response {}: {}".format(flow.request.path, flow.response)) config = resolve_config(flow, "response") if config is None: return required_headers = config.get("headers") if required_headers: headers = {**dict(flow.request.headers), **dict(flow.response.headers)} if not content_matches(None, required_headers, headers): return ctx.log.debug("Match response {}: {}".format(flow.request.path, config)) save = config.get("save", mock_config.get("save")) if save: save_flow(save, flow, "response") replace = config.get("replace") if replace: response = replace.get("response", replace) if response: flow.response = make_response(response, flow.response.status_code, flow.response.content, flow.response.headers) ctx.log.debug("Replace response {}: {}".format( flow.request.path, flow.response)) modify = config.get("modify", []) if isinstance(modify, dict) or isinstance(modify, str): modify = [modify] global_modify = mock_config.get("response", {}).get("*", {}).get("modify") if global_modify: if isinstance(global_modify, dict) or isinstance(global_modify, str): global_modify = [global_modify] modify = global_modify + modify if modify: flow.response.text = content_as_str( modify_content(modify, flow.response.text)) ctx.log.debug("Modify response {}: {}".format(flow.request.path, modify))
def response(flow: http.HTTPFlow): # todo 修改响应的状态码 print("所有状态码%s" % str(flow.response.status_code)) # if '/service/settings/v2/' in flow.request.path: # flow.response = http.HTTPResponse.make(404) # ctx.log.error("响应状态码 %s" % str(flow.response.status_code)) # todo rewrite功能修改字段 忽略非360 搜索地址 if flow.request.host != "www.so.com": return # 将响应text中所有“搜索”替换为“请使用谷歌” text = flow.response.get_text() text = text.replace("搜索", "请使用谷歌") flow.response.set_text(text) flow.response.text = json.dumps(text) # todo 实现Charles中的MapRemote功能 # 加上过滤条件 if "quote.json" in flow.request.pretty_url and "x=" in flow.request.pretty_url: # 把响应数据转化成python对象,保存到data中 data = json.loads(flow.response.content) # 对第一个股票保持原样 data['data']['items'][0] = data['data']['items'][0] # 对第二个股票名字加长一倍 data['data']['items'][1]['quote'][ 'name'] = data['data']['items'][1]['quote']['name'] * 2 # 对第三个股票名字变成空 data['data']['items'][2]['quote']['name'] = None # 把修改后的内容赋值给 response 原始数据格式 flow.response.text = json.dumps(data) # todo 实现Charles中的MapLocal功能 # 修改判断条件 if "quote.json" in flow.request.pretty_url and "x=" in flow.request.pretty_url: # 打开保存在本地的数据文件 with open( "/Users/chenshifeng/MyCode/PythonCode/SFDSZL/interface/quote.json" ) as f: # 创造一个 response flow.response = http.HTTPResponse.make( 200, # (optional) status code # 读取文件中数据作为返回内容 f.read(), # 指定返回数据的类型 {"Content-Type": "application/json"} # (optional) headers ) # todo (MapLocal)修改响应的消息体-通过读取json文件的字符串返给客户端 if flow.request.url.startswith( "https://xxx.x.xxx.com.cn/activityInfo/getPrizeInfo=="): # 读取文件,在当前文件路径下执行脚本,否则需要写文件的绝对路径;不然会找不到该json文件 with open('getStatus.json', 'rb') as f: # 从json文件中读取数据成python对象 res = json.load(f) # 将读取的python对象转成json字符串发送给客户端 flow.response.set_text(json.dumps(res)) ctx.log.info("modify order status") # todo 方法2 (rewrite)字段 修改响应的消息体-直接修改响应字段 if flow.request.url.startswith( "https://activity.x.xxx.com.cn/activityPlugDrawInfo/getPrizeInfo==" ): # 获取响应的json字符串,转成python对象进行解析和修改 response = json.loads(flow.response.get_text()) response['limitCount'] = 1 # 修改完成后,奖python对象转成json字符串,set进请求的响应体重发送给客户端 flow.response.set_text() ctx.log.info('modify limitCount')
def request(flow: http.HTTPFlow): # 发起请求,判断 url 是不是预期值 if "events" in flow.request.pretty_url: # 创造一个 response flow.response = http.HTTPResponse.make( 200, # (optional) status code event, # (optional) content {"Content-Type": "application/json"} # (optional) headers ) # 发起请求,判断 url 是不是预期值 if "eventtypesdistr" in flow.request.pretty_url: # 打开一个保存在本地的文件 with open("data/eventtypedistr.json") as f: # 创造一个 response flow.response = http.HTTPResponse.make( 200, # (optional) status code f.read(), # (optional) content {"Content-Type": "application/json"} # (optional) headers ) # 发起请求,判断 url 是不是预期值 if "traffic" in flow.request.pretty_url: # 打开一个保存在本地的文件 with open("data/traffic.json") as f: # 创造一个 response flow.response = http.HTTPResponse.make( 200, # (optional) status code f.read(), # (optional) content {"Content-Type": "application/json"} # (optional) headers ) # 发起请求,判断 url 是不是预期值 if "srctopn" in flow.request.pretty_url: # 打开一个保存在本地的文件 with open("data/srctopn.json") as f: # 创造一个 response flow.response = http.HTTPResponse.make( 200, # (optional) status code f.read(), # (optional) content {"Content-Type": "application/json"} # (optional) headers ) # 发起请求,判断 url 是不是预期值 if "/topn/?chart=bar&unit=bps&sortby=In" in flow.request.pretty_url: # 打开一个保存在本地的文件 with open("data/topn.json") as f: # 创造一个 response flow.response = http.HTTPResponse.make( 200, # (optional) status code f.read(), # (optional) content {"Content-Type": "application/json"} # (optional) headers ) # 发起请求,判断 url 是不是预期值 if "eventdistr" in flow.request.pretty_url: # 打开一个保存在本地的文件 with open("data/eventdistr.json") as f: # 创造一个 response flow.response = http.HTTPResponse.make( 200, # (optional) status code f.read(), # (optional) content {"Content-Type": "application/json"} # (optional) headers ) # 发起请求,判断 url 是不是预期值 if "srctopngeodist" in flow.request.pretty_url: # 打开一个保存在本地的文件 with open("data/srctopngeodist.json") as f: # 创造一个 response flow.response = http.HTTPResponse.make( 200, # (optional) status code f.read(), # (optional) content {"Content-Type": "application/json"} # (optional) headers ) # 发起请求,判断 url 是不是预期值 if "abnormaltopn" in flow.request.pretty_url: # 打开一个保存在本地的文件 with open("data/adnormaltopn.json") as f: # 创造一个 response flow.response = http.HTTPResponse.make( 200, # (optional) status code f.read(), # (optional) content {"Content-Type": "application/json"} # (optional) headers )
def response(flow: http.HTTPFlow) -> None: if random.randint(1, 100) < percentage: ctx.log.warn('>>> Down ' + flow.request.url) flow.response = http.HTTPResponse.make(503, '', {})
def request(self, flow: http.HTTPFlow): """Handle HTTP Request.""" js_tool = False full_url = flow.request.pretty_url.split("?")[0] if "?JSTool=" in flow.request.pretty_url: full_url += "?JSTool" js_tool = True hashed_url = hashlib.sha256(full_url.encode()).hexdigest() cursor = self.cnx.cursor() cursor.execute( "SELECT `file_name` FROM resource_location " "WHERE `hashed_url` LIKE '%s'" % (hashed_url) ) result = cursor.fetchone() if result: # found in db file_name = str(result[0]) file_path = self.path + "/cache/" + file_name if js_tool: # headers with open(file_path + '.h', 'rb') as temp_file: temp_headers = pickle.load(temp_file) headers_dict = {} for attr in list(temp_headers): headers_dict[attr] = temp_headers[attr] # make simplification with open(file_path + '.c', 'rb') as temp_file: temp_content = pickle.load(temp_file) html = str(BeautifulSoup(temp_content, 'html.parser')) active = flow.request.pretty_url.split( "?JSTool=")[-1].split("_") active = active[1:] for num in active: index = html.find("<!--script" + str(num) + "\n") html = html.replace( "<!--script" + str(num) + "\n", "<script") html = html[0:index] + \ html[index:].replace("-->\n", "</script>", 1) # return response flow.response = http.HTTPResponse.make( 200, # (optional) status code html, # (optional) content headers_dict) print("*" * 30) print(full_url, "served from cache file", file_name) else: # delete existing entries cursor.execute( "DELETE FROM resource_location " "WHERE `hashed_url` LIKE '%s'" % (hashed_url) ) try: os.remove(file_path + '.h') os.remove(file_path + '.c') except FileNotFoundError: pass elif js_tool: full_url = flow.request.pretty_url.split("?")[0] hashed_url = hashlib.sha256(full_url.encode()).hexdigest() cursor.execute( "SELECT `file_name` FROM resource_location " "WHERE `hashed_url` LIKE '%s'" % (hashed_url) ) result = cursor.fetchone() if result: # Add version with scripts removed to cache file_name = str(result[0]) file_path = self.path + "/cache/" + file_name with open(file_path + '.h', 'rb') as temp_file: temp_headers = pickle.load(temp_file) headers_dict = {} for attr in list(temp_headers): headers_dict[attr] = temp_headers[attr] with open(file_path + '.c', 'rb') as temp_file: temp_content = pickle.load(temp_file) html = str(BeautifulSoup(temp_content, 'html.parser')) # Remove comments while "<!--" in html: start_index = html.find("<!--") end_index = html.find("-->") + 3 html = html.replace(html[start_index:end_index], "") cnt = 1 while "<script" in html: html = html.replace( "<script", "\n<!--script" + str(cnt) + "\n", 1) html = html.replace("</script>", "\n-->\n", 1) print("cnt =", cnt) cnt += 1 # return response flow.response = http.HTTPResponse.make( 200, # (optional) status code html, # (optional) content headers_dict) # add to database full_url += "?JSTool" hashed_url = hashlib.sha256(full_url.encode()).hexdigest() file_name = self.random_filename() file_path = self.path + "/cache/" + file_name cursor.execute( "INSERT INTO resource_location " "(`hashed_url`, `full_url`, `file_name`, datetime) " "VALUES ('%s', '%s', '%s', NOW())" % (hashed_url, full_url, file_name) ) self.cnx.commit() # write to file with open(file_path + '.h', 'wb') as temp_file: pickle.dump(flow.response.headers, temp_file) with open(file_path + '.c', 'wb') as temp_file: pickle.dump(flow.response.content, temp_file) cursor.close()
def request(self, flow: http.HTTPFlow): if flow.request.path == "/cspscannerreport": flow.response = http.HTTPResponse.make(200)
def request(flow: http.HTTPFlow): if flow.request.pretty_url == "https://www.baidu.com/": flow.response = http.HTTPResponse.make(200, b"Hello Word", {"Content-Type": "text/html"})
def request(flow: http.HTTPFlow) -> None: if (flow.request.pretty_host == "yatsun.me"): flow.response = http.HTTPResponse.make( 200, '<html><head></head><body><h1>Test</h1></body></html>', {"Content-Type": "text/html"})
def request(flow: http.HTTPFlow): if "baidu.com" in flow.request.pretty_url: # if flow.request.pretty_url == "http://example.com/path": flow.response = http.HTTPResponse.make(200, b"Hello World", {"Content-Type": "text/html"})
def reply_from_proxy(flow: HTTPFlow): flow.response = Response.make(418)