class WASEProxyHandler(ProxyHandler):
"""Intercepts HTTP(S) requests/responses, extracts data and feeds ElasticSearch"""
def mitm_request(self, data):
r = HTTPRequest(data)
# determine url
if self.is_connect:
scheme = "https"
else:
scheme = "http"
url = scheme + "://" + self.hostname
if scheme == "http" and int(self.port) != 80 or scheme == "https" and int(self.port) != 443:
url += ":" + self.port
url += self.path
if args.verbose:
print(url)
self.doc = DocHTTPRequestResponse(host=self.hostname, port=int(self.port), protocol=scheme)
self.doc.meta.index = args.index
self.doc.request.url = url
self.doc.request.requestline = r.requestline
self.doc.request.method = r.command
self.doc.host = self.hostname
self.doc.port = int(self.port)
self.doc.protocol = scheme
return data
def mitm_response(self, data):
lines = data.split("\r\n")
r = HTTPResponse(FakeSocket(data))
r.begin()
# response line
self.doc.response.status = r.status
self.doc.response.responseline = lines[0]
# headers
ct = ""
cookies = list()
for header in r.getheaders():
name = header[0]
value = header[1]
self.doc.add_parsed_response_header(name, value)
if name == "content-type":
ct = value
elif name == "set-cookie":
cookies.append(value)
# content type
try:
m = reContentType.search(ct)
self.doc.response.content_type = m.group(1)
except:
pass
# cookies
for cookie in cookies:
# TODO: the following code extracts only partial cookie data - check/rewrite
try:
pc = SimpleCookie(cookie)
for name in pc.keys():
c = pc[name]
try:
value = c.value
except AttributeError:
value = None
try:
domain = c.domain
except AttributeError:
domain = None
try:
path = c.path
except AttributeError:
path = None
try:
exp = c.expires
except AttributeError:
exp = None
self.doc.add_response_cookie(name, value, domain, path, exp)
except:
pass
# body
bodybytes = r.read()
self.doc.response.body = bodybytes.decode(args.charset, args.encodingerrors)
try:
self.doc.save(storeResponseBody)
except:
pass
return data
def saveToES(self, msg):
httpService = msg.getHttpService()
doc = DocHTTPRequestResponse(protocol=httpService.getProtocol(), host=httpService.getHost(), port=httpService.getPort())
request = msg.getRequest()
response = msg.getResponse()
if request:
iRequest = self.helpers.analyzeRequest(msg)
doc.request.method = iRequest.getMethod()
doc.request.url = iRequest.getUrl().toString()
headers = iRequest.getHeaders()
for header in headers:
try:
doc.add_request_header(header)
except:
doc.request.requestline = header
parameters = iRequest.getParameters()
for parameter in parameters:
ptype = parameter.getType()
if ptype == IParameter.PARAM_URL:
typename = "url"
elif ptype == IParameter.PARAM_BODY:
typename = "body"
elif ptype == IParameter.PARAM_COOKIE:
typename = "cookie"
elif ptype == IParameter.PARAM_XML:
typename = "xml"
elif ptype == IParameter.PARAM_XML_ATTR:
typename = "xmlattr"
elif ptype == IParameter.PARAM_MULTIPART_ATTR:
typename = "multipartattr"
elif ptype == IParameter.PARAM_JSON:
typename = "json"
else:
typename = "unknown"
name = parameter.getName()
value = parameter.getValue()
doc.add_request_parameter(typename, name, value)
ctype = iRequest.getContentType()
if ctype == IRequestInfo.CONTENT_TYPE_NONE:
doc.request.content_type = "none"
elif ctype == IRequestInfo.CONTENT_TYPE_URL_ENCODED:
doc.request.content_type = "urlencoded"
elif ctype == IRequestInfo.CONTENT_TYPE_MULTIPART:
doc.request.content_type = "multipart"
elif ctype == IRequestInfo.CONTENT_TYPE_XML:
doc.request.content_type = "xml"
elif ctype == IRequestInfo.CONTENT_TYPE_JSON:
doc.request.content_type = "json"
elif ctype == IRequestInfo.CONTENT_TYPE_AMF:
doc.request.content_type = "amf"
else:
doc.request.content_type = "unknown"
bodyOffset = iRequest.getBodyOffset()
doc.request.body = request[bodyOffset:].tostring().decode("ascii", "replace")
if response:
iResponse = self.helpers.analyzeResponse(response)
doc.response.status = iResponse.getStatusCode()
doc.response.content_type = iResponse.getStatedMimeType()
doc.response.inferred_content_type = iResponse.getInferredMimeType()
headers = iResponse.getHeaders()
for header in headers:
try:
doc.add_response_header(header)
except:
doc.response.responseline = header
cookies = iResponse.getCookies()
for cookie in cookies:
expCookie = cookie.getExpiration()
expiration = None
if expCookie:
try:
expiration = datetime.fromtimestamp(expCookie.time / 1000)
except:
pass
doc.add_response_cookie(cookie.getName(), cookie.getValue(), cookie.getExpiration(), cookie.getPath(), expiration)
bodyOffset = iResponse.getBodyOffset()
doc.response.body = response[bodyOffset:].tostring().decode("ascii", "replace")
doc.save()
def genESDoc(self, msg, timeStampFromResponse=False):
httpService = msg.getHttpService()
doc = DocHTTPRequestResponse(protocol=httpService.getProtocol(),
host=httpService.getHost(),
port=httpService.getPort())
doc.meta.index = self.confESIndex
request = msg.getRequest()
response = msg.getResponse()
if request:
iRequest = self.helpers.analyzeRequest(msg)
doc.request.method = iRequest.getMethod()
doc.request.url = iRequest.getUrl().toString()
headers = iRequest.getHeaders()
for header in headers:
try:
doc.add_request_header(header)
except:
doc.request.requestline = header
parameters = iRequest.getParameters()
for parameter in parameters:
ptype = parameter.getType()
if ptype == IParameter.PARAM_URL:
typename = "url"
elif ptype == IParameter.PARAM_BODY:
typename = "body"
elif ptype == IParameter.PARAM_COOKIE:
typename = "cookie"
elif ptype == IParameter.PARAM_XML:
typename = "xml"
elif ptype == IParameter.PARAM_XML_ATTR:
typename = "xmlattr"
elif ptype == IParameter.PARAM_MULTIPART_ATTR:
typename = "multipartattr"
elif ptype == IParameter.PARAM_JSON:
typename = "json"
else:
typename = "unknown"
name = parameter.getName()
value = parameter.getValue()
doc.add_request_parameter(typename, name, value)
ctype = iRequest.getContentType()
if ctype == IRequestInfo.CONTENT_TYPE_NONE:
doc.request.content_type = "none"
elif ctype == IRequestInfo.CONTENT_TYPE_URL_ENCODED:
doc.request.content_type = "urlencoded"
elif ctype == IRequestInfo.CONTENT_TYPE_MULTIPART:
doc.request.content_type = "multipart"
elif ctype == IRequestInfo.CONTENT_TYPE_XML:
doc.request.content_type = "xml"
elif ctype == IRequestInfo.CONTENT_TYPE_JSON:
doc.request.content_type = "json"
elif ctype == IRequestInfo.CONTENT_TYPE_AMF:
doc.request.content_type = "amf"
else:
doc.request.content_type = "unknown"
bodyOffset = iRequest.getBodyOffset()
doc.request.body = request[bodyOffset:].tostring().decode(
"ascii", "replace")
if response:
iResponse = self.helpers.analyzeResponse(response)
doc.response.status = iResponse.getStatusCode()
doc.response.content_type = iResponse.getStatedMimeType()
doc.response.inferred_content_type = iResponse.getInferredMimeType(
)
headers = iResponse.getHeaders()
dateHeader = None
for header in headers:
try:
doc.add_response_header(header)
match = reDateHeader.match(header)
if match:
dateHeader = match.group(1)
except:
doc.response.responseline = header
cookies = iResponse.getCookies()
for cookie in cookies:
expCookie = cookie.getExpiration()
expiration = None
if expCookie:
try:
expiration = str(
datetime.fromtimestamp(expCookie.time / 1000))
except:
pass
doc.add_response_cookie(cookie.getName(), cookie.getValue(),
cookie.getDomain(), cookie.getPath(),
expiration)
bodyOffset = iResponse.getBodyOffset()
doc.response.body = response[bodyOffset:].tostring().decode(
"ascii", "replace")
if timeStampFromResponse:
if dateHeader:
try:
doc.timestamp = datetime.fromtimestamp(
mktime_tz(parsedate_tz(dateHeader)),
tz) # try to use date from response header "Date"
self.lastTimestamp = doc.timestamp
except:
doc.timestamp = self.lastTimestamp # fallback: last stored timestamp. Else: now
return doc
def genESDoc(self, msg, timeStampFromResponse=False):
httpService = msg.getHttpService()
doc = DocHTTPRequestResponse(protocol=httpService.getProtocol(), host=httpService.getHost(), port=httpService.getPort())
doc.meta.index = self.confESIndex
request = msg.getRequest()
response = msg.getResponse()
if request:
iRequest = self.helpers.analyzeRequest(msg)
doc.request.method = iRequest.getMethod()
doc.request.url = iRequest.getUrl().toString()
headers = iRequest.getHeaders()
for header in headers:
try:
doc.add_request_header(header)
except:
doc.request.requestline = header
parameters = iRequest.getParameters()
for parameter in parameters:
ptype = parameter.getType()
if ptype == IParameter.PARAM_URL:
typename = "url"
elif ptype == IParameter.PARAM_BODY:
typename = "body"
elif ptype == IParameter.PARAM_COOKIE:
typename = "cookie"
elif ptype == IParameter.PARAM_XML:
typename = "xml"
elif ptype == IParameter.PARAM_XML_ATTR:
typename = "xmlattr"
elif ptype == IParameter.PARAM_MULTIPART_ATTR:
typename = "multipartattr"
elif ptype == IParameter.PARAM_JSON:
typename = "json"
else:
typename = "unknown"
name = parameter.getName()
value = parameter.getValue()
doc.add_request_parameter(typename, name, value)
ctype = iRequest.getContentType()
if ctype == IRequestInfo.CONTENT_TYPE_NONE:
doc.request.content_type = "none"
elif ctype == IRequestInfo.CONTENT_TYPE_URL_ENCODED:
doc.request.content_type = "urlencoded"
elif ctype == IRequestInfo.CONTENT_TYPE_MULTIPART:
doc.request.content_type = "multipart"
elif ctype == IRequestInfo.CONTENT_TYPE_XML:
doc.request.content_type = "xml"
elif ctype == IRequestInfo.CONTENT_TYPE_JSON:
doc.request.content_type = "json"
elif ctype == IRequestInfo.CONTENT_TYPE_AMF:
doc.request.content_type = "amf"
else:
doc.request.content_type = "unknown"
bodyOffset = iRequest.getBodyOffset()
doc.request.body = request[bodyOffset:].tostring().decode("ascii", "replace")
if response:
iResponse = self.helpers.analyzeResponse(response)
doc.response.status = iResponse.getStatusCode()
doc.response.content_type = iResponse.getStatedMimeType()
doc.response.inferred_content_type = iResponse.getInferredMimeType()
headers = iResponse.getHeaders()
dateHeader = None
for header in headers:
try:
doc.add_response_header(header)
match = reDateHeader.match(header)
if match:
dateHeader = match.group(1)
except:
doc.response.responseline = header
cookies = iResponse.getCookies()
for cookie in cookies:
expCookie = cookie.getExpiration()
expiration = None
if expCookie:
try:
expiration = str(datetime.fromtimestamp(expCookie.time / 1000))
except:
pass
doc.add_response_cookie(cookie.getName(), cookie.getValue(), cookie.getDomain(), cookie.getPath(), expiration)
bodyOffset = iResponse.getBodyOffset()
doc.response.body = response[bodyOffset:].tostring().decode("ascii", "replace")
if timeStampFromResponse:
if dateHeader:
try:
doc.timestamp = datetime.fromtimestamp(mktime_tz(parsedate_tz(dateHeader)), tz) # try to use date from response header "Date"
self.lastTimestamp = doc.timestamp
except:
doc.timestamp = self.lastTimestamp # fallback: last stored timestamp. Else: now
return doc
class WASEProxyHandler(ProxyHandler):
"""Intercepts HTTP(S) requests/responses, extracts data and feeds ElasticSearch"""
def mitm_request(self, data):
# Initialize ES connection and index
res = connections.create_connection(hosts=[args.elasticsearch])
idx = Index(args.index)
idx.doc_type(DocHTTPRequestResponse)
try:
DocHTTPRequestResponse.init()
idx.create()
except:
pass
r = HTTPRequest(data)
# determine url
if self.is_connect:
scheme = "https"
else:
scheme = "http"
url = scheme + "://" + self.hostname
if scheme == "http" and int(self.port) != 80 or scheme == "https" and int(self.port) != 443:
url += ":" + str(self.port)
url += self.path
if args.verbose:
print(url)
self.doc = DocHTTPRequestResponse(host=self.hostname, port=int(self.port), protocol=scheme)
self.doc.meta.index = args.index
self.doc.request.url = url
self.doc.request.requestline = r.requestline
self.doc.request.method = r.command
self.doc.host = self.hostname
self.doc.port = int(self.port)
self.doc.protocol = scheme
return data
def mitm_response(self, data):
lines = data.split("\r\n")
r = HTTPResponse(FakeSocket(data))
r.begin()
# response line
self.doc.response.status = r.status
self.doc.response.responseline = lines[0].decode(args.charset, args.encodingerrors)
# headers
ct = ""
cookies = list()
for header in r.getheaders():
name = header[0].decode(args.charset, args.encodingerrors)
value = header[1].decode(args.charset, args.encodingerrors)
self.doc.add_parsed_response_header(name, value)
if name == "content-type":
ct = value
elif name == "set-cookie":
cookies.append(value)
# content type
try:
m = reContentType.search(ct)
self.doc.response.content_type = m.group(1)
except:
pass
# cookies
for cookie in cookies:
# TODO: the following code extracts only partial cookie data - check/rewrite
try:
pc = SimpleCookie(cookie)
for name in pc.keys():
c = pc[name]
try:
value = c.value
except AttributeError:
value = None
try:
domain = c.domain
except AttributeError:
domain = None
try:
path = c.path
except AttributeError:
path = None
try:
exp = c.expires
except AttributeError:
exp = None
self.doc.add_response_cookie(name, value, domain, path, exp)
except:
pass
# body
bodybytes = r.read()
self.doc.response.body = bodybytes.decode(args.charset, args.encodingerrors)
self.doc.save(storeResponseBody)
return data
#idx.create()
DocHTTPRequestResponse.init()
d = DocHTTPRequestResponse(
protocol="http",
host="foobar.com",
port=80
)
d.add_request_header("User-Agent: foobar")
d.add_request_parameter("url", "id", "123")
d.add_request_parameter("url", "doc", "234")
d.add_response_header("X-Content-Type-Options: nosniff")
d.add_response_header("X-Frame-Options: DENY")
d.add_response_header("X-XSS-Protection: 1; mode=block")
d.add_response_cookie("SESSIONID", "foobar1234")
d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now())
d.response.body = "This is a test!"
d.request.method = "GET"
d.save()
d = DocHTTPRequestResponse(
protocol="http",
host="foobar.com",
port=80
)
d.add_request_header("User-Agent: foobar")
d.add_request_parameter("url", "id", "123")
d.add_request_parameter("url", "doc", "456")
d.add_response_header("X-Frame-Options: SAMEORIGIN")
d.add_response_cookie("SESSIONID", "foobar1234")
connections.create_connection(hosts=["localhost"])
idx = Index("test")
idx.doc_type(DocHTTPRequestResponse)
#idx.create()
DocHTTPRequestResponse.init()
d = DocHTTPRequestResponse(protocol="http", host="foobar.com", port=80)
d.add_request_header("User-Agent: foobar")
d.add_request_parameter("url", "id", "123")
d.add_request_parameter("url", "doc", "234")
d.add_response_header("X-Content-Type-Options: nosniff")
d.add_response_header("X-Frame-Options: DENY")
d.add_response_header("X-XSS-Protection: 1; mode=block")
d.add_response_cookie("SESSIONID", "foobar1234")
d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now())
d.response.body = "This is a test!"
d.request.method = "GET"
d.save()
d = DocHTTPRequestResponse(protocol="http", host="foobar.com", port=80)
d.add_request_header("User-Agent: foobar")
d.add_request_parameter("url", "id", "123")
d.add_request_parameter("url", "doc", "456")
d.add_response_header("X-Frame-Options: SAMEORIGIN")
d.add_response_cookie("SESSIONID", "foobar1234")
d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now())
d.request.method = "GET"
d.response.body = "This is a test!"
d.save()