def saveToES(self, msg):
httpService = msg.getHttpService()
doc = DocHTTPRequestResponse(protocol=httpService.getProtocol(), host=httpService.getHost(), port=httpService.getPort())
request = msg.getRequest()
response = msg.getResponse()
if request:
iRequest = self.helpers.analyzeRequest(msg)
doc.request.method = iRequest.getMethod()
doc.request.url = iRequest.getUrl().toString()
headers = iRequest.getHeaders()
for header in headers:
try:
doc.add_request_header(header)
except:
doc.request.requestline = header
parameters = iRequest.getParameters()
for parameter in parameters:
ptype = parameter.getType()
if ptype == IParameter.PARAM_URL:
typename = "url"
elif ptype == IParameter.PARAM_BODY:
typename = "body"
elif ptype == IParameter.PARAM_COOKIE:
typename = "cookie"
elif ptype == IParameter.PARAM_XML:
typename = "xml"
elif ptype == IParameter.PARAM_XML_ATTR:
typename = "xmlattr"
elif ptype == IParameter.PARAM_MULTIPART_ATTR:
typename = "multipartattr"
elif ptype == IParameter.PARAM_JSON:
typename = "json"
else:
typename = "unknown"
name = parameter.getName()
value = parameter.getValue()
doc.add_request_parameter(typename, name, value)
ctype = iRequest.getContentType()
if ctype == IRequestInfo.CONTENT_TYPE_NONE:
doc.request.content_type = "none"
elif ctype == IRequestInfo.CONTENT_TYPE_URL_ENCODED:
doc.request.content_type = "urlencoded"
elif ctype == IRequestInfo.CONTENT_TYPE_MULTIPART:
doc.request.content_type = "multipart"
elif ctype == IRequestInfo.CONTENT_TYPE_XML:
doc.request.content_type = "xml"
elif ctype == IRequestInfo.CONTENT_TYPE_JSON:
doc.request.content_type = "json"
elif ctype == IRequestInfo.CONTENT_TYPE_AMF:
doc.request.content_type = "amf"
else:
doc.request.content_type = "unknown"
bodyOffset = iRequest.getBodyOffset()
doc.request.body = request[bodyOffset:].tostring().decode("ascii", "replace")
if response:
iResponse = self.helpers.analyzeResponse(response)
doc.response.status = iResponse.getStatusCode()
doc.response.content_type = iResponse.getStatedMimeType()
doc.response.inferred_content_type = iResponse.getInferredMimeType()
headers = iResponse.getHeaders()
for header in headers:
try:
doc.add_response_header(header)
except:
doc.response.responseline = header
cookies = iResponse.getCookies()
for cookie in cookies:
expCookie = cookie.getExpiration()
expiration = None
if expCookie:
try:
expiration = datetime.fromtimestamp(expCookie.time / 1000)
except:
pass
doc.add_response_cookie(cookie.getName(), cookie.getValue(), cookie.getExpiration(), cookie.getPath(), expiration)
bodyOffset = iResponse.getBodyOffset()
doc.response.body = response[bodyOffset:].tostring().decode("ascii", "replace")
doc.save()
def genESDoc(self, msg, timeStampFromResponse=False):
httpService = msg.getHttpService()
doc = DocHTTPRequestResponse(protocol=httpService.getProtocol(),
host=httpService.getHost(),
port=httpService.getPort())
doc.meta.index = self.confESIndex
request = msg.getRequest()
response = msg.getResponse()
if request:
iRequest = self.helpers.analyzeRequest(msg)
doc.request.method = iRequest.getMethod()
doc.request.url = iRequest.getUrl().toString()
headers = iRequest.getHeaders()
for header in headers:
try:
doc.add_request_header(header)
except:
doc.request.requestline = header
parameters = iRequest.getParameters()
for parameter in parameters:
ptype = parameter.getType()
if ptype == IParameter.PARAM_URL:
typename = "url"
elif ptype == IParameter.PARAM_BODY:
typename = "body"
elif ptype == IParameter.PARAM_COOKIE:
typename = "cookie"
elif ptype == IParameter.PARAM_XML:
typename = "xml"
elif ptype == IParameter.PARAM_XML_ATTR:
typename = "xmlattr"
elif ptype == IParameter.PARAM_MULTIPART_ATTR:
typename = "multipartattr"
elif ptype == IParameter.PARAM_JSON:
typename = "json"
else:
typename = "unknown"
name = parameter.getName()
value = parameter.getValue()
doc.add_request_parameter(typename, name, value)
ctype = iRequest.getContentType()
if ctype == IRequestInfo.CONTENT_TYPE_NONE:
doc.request.content_type = "none"
elif ctype == IRequestInfo.CONTENT_TYPE_URL_ENCODED:
doc.request.content_type = "urlencoded"
elif ctype == IRequestInfo.CONTENT_TYPE_MULTIPART:
doc.request.content_type = "multipart"
elif ctype == IRequestInfo.CONTENT_TYPE_XML:
doc.request.content_type = "xml"
elif ctype == IRequestInfo.CONTENT_TYPE_JSON:
doc.request.content_type = "json"
elif ctype == IRequestInfo.CONTENT_TYPE_AMF:
doc.request.content_type = "amf"
else:
doc.request.content_type = "unknown"
bodyOffset = iRequest.getBodyOffset()
doc.request.body = request[bodyOffset:].tostring().decode(
"ascii", "replace")
if response:
iResponse = self.helpers.analyzeResponse(response)
doc.response.status = iResponse.getStatusCode()
doc.response.content_type = iResponse.getStatedMimeType()
doc.response.inferred_content_type = iResponse.getInferredMimeType(
)
headers = iResponse.getHeaders()
dateHeader = None
for header in headers:
try:
doc.add_response_header(header)
match = reDateHeader.match(header)
if match:
dateHeader = match.group(1)
except:
doc.response.responseline = header
cookies = iResponse.getCookies()
for cookie in cookies:
expCookie = cookie.getExpiration()
expiration = None
if expCookie:
try:
expiration = str(
datetime.fromtimestamp(expCookie.time / 1000))
except:
pass
doc.add_response_cookie(cookie.getName(), cookie.getValue(),
cookie.getDomain(), cookie.getPath(),
expiration)
bodyOffset = iResponse.getBodyOffset()
doc.response.body = response[bodyOffset:].tostring().decode(
"ascii", "replace")
if timeStampFromResponse:
if dateHeader:
try:
doc.timestamp = datetime.fromtimestamp(
mktime_tz(parsedate_tz(dateHeader)),
tz) # try to use date from response header "Date"
self.lastTimestamp = doc.timestamp
except:
doc.timestamp = self.lastTimestamp # fallback: last stored timestamp. Else: now
return doc
connections.create_connection(hosts=["localhost"])
idx = Index("test")
idx.doc_type(DocHTTPRequestResponse)
#idx.create()
DocHTTPRequestResponse.init()
d = DocHTTPRequestResponse(
protocol="http",
host="foobar.com",
port=80
)
d.add_request_header("User-Agent: foobar")
d.add_request_parameter("url", "id", "123")
d.add_request_parameter("url", "doc", "234")
d.add_response_header("X-Content-Type-Options: nosniff")
d.add_response_header("X-Frame-Options: DENY")
d.add_response_header("X-XSS-Protection: 1; mode=block")
d.add_response_cookie("SESSIONID", "foobar1234")
d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now())
d.response.body = "This is a test!"
d.request.method = "GET"
d.save()
d = DocHTTPRequestResponse(
protocol="http",
host="foobar.com",
port=80
)
def genESDoc(self, msg, timeStampFromResponse=False):
httpService = msg.getHttpService()
doc = DocHTTPRequestResponse(protocol=httpService.getProtocol(), host=httpService.getHost(), port=httpService.getPort())
doc.meta.index = self.confESIndex
request = msg.getRequest()
response = msg.getResponse()
if request:
iRequest = self.helpers.analyzeRequest(msg)
doc.request.method = iRequest.getMethod()
doc.request.url = iRequest.getUrl().toString()
headers = iRequest.getHeaders()
for header in headers:
try:
doc.add_request_header(header)
except:
doc.request.requestline = header
parameters = iRequest.getParameters()
for parameter in parameters:
ptype = parameter.getType()
if ptype == IParameter.PARAM_URL:
typename = "url"
elif ptype == IParameter.PARAM_BODY:
typename = "body"
elif ptype == IParameter.PARAM_COOKIE:
typename = "cookie"
elif ptype == IParameter.PARAM_XML:
typename = "xml"
elif ptype == IParameter.PARAM_XML_ATTR:
typename = "xmlattr"
elif ptype == IParameter.PARAM_MULTIPART_ATTR:
typename = "multipartattr"
elif ptype == IParameter.PARAM_JSON:
typename = "json"
else:
typename = "unknown"
name = parameter.getName()
value = parameter.getValue()
doc.add_request_parameter(typename, name, value)
ctype = iRequest.getContentType()
if ctype == IRequestInfo.CONTENT_TYPE_NONE:
doc.request.content_type = "none"
elif ctype == IRequestInfo.CONTENT_TYPE_URL_ENCODED:
doc.request.content_type = "urlencoded"
elif ctype == IRequestInfo.CONTENT_TYPE_MULTIPART:
doc.request.content_type = "multipart"
elif ctype == IRequestInfo.CONTENT_TYPE_XML:
doc.request.content_type = "xml"
elif ctype == IRequestInfo.CONTENT_TYPE_JSON:
doc.request.content_type = "json"
elif ctype == IRequestInfo.CONTENT_TYPE_AMF:
doc.request.content_type = "amf"
else:
doc.request.content_type = "unknown"
bodyOffset = iRequest.getBodyOffset()
doc.request.body = request[bodyOffset:].tostring().decode("ascii", "replace")
if response:
iResponse = self.helpers.analyzeResponse(response)
doc.response.status = iResponse.getStatusCode()
doc.response.content_type = iResponse.getStatedMimeType()
doc.response.inferred_content_type = iResponse.getInferredMimeType()
headers = iResponse.getHeaders()
dateHeader = None
for header in headers:
try:
doc.add_response_header(header)
match = reDateHeader.match(header)
if match:
dateHeader = match.group(1)
except:
doc.response.responseline = header
cookies = iResponse.getCookies()
for cookie in cookies:
expCookie = cookie.getExpiration()
expiration = None
if expCookie:
try:
expiration = str(datetime.fromtimestamp(expCookie.time / 1000))
except:
pass
doc.add_response_cookie(cookie.getName(), cookie.getValue(), cookie.getDomain(), cookie.getPath(), expiration)
bodyOffset = iResponse.getBodyOffset()
doc.response.body = response[bodyOffset:].tostring().decode("ascii", "replace")
if timeStampFromResponse:
if dateHeader:
try:
doc.timestamp = datetime.fromtimestamp(mktime_tz(parsedate_tz(dateHeader)), tz) # try to use date from response header "Date"
self.lastTimestamp = doc.timestamp
except:
doc.timestamp = self.lastTimestamp # fallback: last stored timestamp. Else: now
return doc
from doc_HttpRequestResponse import DocHTTPRequestResponse
from elasticsearch_dsl.connections import connections
from elasticsearch_dsl import Index
from datetime import datetime
connections.create_connection(hosts=["localhost"])
idx = Index("test")
idx.doc_type(DocHTTPRequestResponse)
#idx.create()
DocHTTPRequestResponse.init()
d = DocHTTPRequestResponse(protocol="http", host="foobar.com", port=80)
d.add_request_header("User-Agent: foobar")
d.add_request_parameter("url", "id", "123")
d.add_request_parameter("url", "doc", "234")
d.add_response_header("X-Content-Type-Options: nosniff")
d.add_response_header("X-Frame-Options: DENY")
d.add_response_header("X-XSS-Protection: 1; mode=block")
d.add_response_cookie("SESSIONID", "foobar1234")
d.add_response_cookie("foo", "bar", "foobar.com", "/foo", datetime.now())
d.response.body = "This is a test!"
d.request.method = "GET"
d.save()
d = DocHTTPRequestResponse(protocol="http", host="foobar.com", port=80)
d.add_request_header("User-Agent: foobar")
d.add_request_parameter("url", "id", "123")
d.add_request_parameter("url", "doc", "456")
d.add_response_header("X-Frame-Options: SAMEORIGIN")
