escape(p.request['url'])),
mtime=timestamp, size=size
)
## Update the inode again:
#new_inode = new_inode % inode_id
## This updates the inode table with the new inode
#self.fsfd.VFSCreate(None,new_inode,
# None, update_only = True,
# inode_id = inode_id
# )
## Store information about this request in the
## http table:
host = p.request.get("host",IP2str(stream.dest_ip))
url = HTML.url_unquote(p.request.get("url"))
try:
date = p.response["date"]
date = Time.parse(date, case=self.case, evidence_tz=None)
except (KeyError,ValueError):
date = 0
## Two forms for the referrer:
referer = p.request.get('referer', p.request.get('referrer',''))
if not url.startswith("http://") and not url.startswith("ftp://"):
url = "http://%s%s" % (host, url)
## Not sure if we really care about this?
## Find referred page:
## parent = 0
dbh = DB.DBO(self.case)
def process_stream(self, stream, factories):
""" We look for HTTP requests to identify the stream. This
allows us to processes HTTP connections on unusual ports. This
situation might arise if HTTP proxies are used for example.
"""
if stream.reverse:
combined_inode = "I%s|S%s/%s" % (stream.fd.name, stream.inode_id,
stream.reverse)
try:
fd = self.fsfd.open(inode=combined_inode)
## If we cant open the combined stream, we quit (This could
## happen if we are trying to operate on a combined stream
## already
except IOError:
return
else:
fd = stream
p = HTTP(fd, self.fsfd)
## Check that this is really HTTP
if not p.identify():
return
pyflaglog.log(pyflaglog.DEBUG, "Openning %s for HTTP" % combined_inode)
## Iterate over all the messages in this connection
for f in p.parse():
if not f: continue
offset, size = f
## Create the VFS node:
new_inode = "%s|H%s:%s" % (combined_inode, offset, size)
try:
if 'chunked' in p.response['transfer-encoding']:
new_inode += "|c0"
except KeyError:
pass
try:
if 'gzip' in p.response['content-encoding']:
new_inode += "|G1"
except KeyError:
pass
try:
if 'deflate' in p.response['content-encoding']:
new_inode += "|d1"
except KeyError:
pass
## stream.ts_sec is already formatted in DB format
## need to convert back to utc/gmt as paths are UTC
timestamp = fd.get_packet_ts(offset)
ds_timestamp = Time.convert(timestamp,
case=self.case,
evidence_tz="UTC")
try:
date_str = ds_timestamp.split(" ")[0]
except:
date_str = stream.ts_sec.split(" ")[0]
path, inode, inode_id = self.fsfd.lookup(inode=combined_inode)
## Try to put the HTTP inodes at the mount point. FIXME:
## This should not be needed when a http stats viewer is
## written.
path = posixpath.normpath(path + "/../../../../../")
inode_id = self.fsfd.VFSCreate(
None,
new_inode,
"%s/HTTP/%s/%s" % (path, date_str, escape(p.request['url'])),
mtime=timestamp,
size=size)
## Update the inode again:
#new_inode = new_inode % inode_id
## This updates the inode table with the new inode
#self.fsfd.VFSCreate(None,new_inode,
# None, update_only = True,
# inode_id = inode_id
# )
## Store information about this request in the
## http table:
host = p.request.get("host", IP2str(stream.dest_ip))
url = HTML.url_unquote(p.request.get("url"))
try:
date = p.response["date"]
date = Time.parse(date, case=self.case, evidence_tz=None)
except (KeyError, ValueError):
date = 0
## Two forms for the referrer:
referer = p.request.get('referer', p.request.get('referrer', ''))
if not url.startswith("http://") and not url.startswith("ftp://"):
url = "http://%s%s" % (host, url)
## Not sure if we really care about this?
## Find referred page:
## parent = 0
dbh = DB.DBO(self.case)
## if referer:
## dbh.execute("select inode_id from http where url=%r order by inode_id desc limit 1", referer)
## row = dbh.fetch()
## ## If there is no referrer we just make a psuedo entry
## if not row:
## ## Find out the host
## m=re.match("(http://|ftp://)([^/]+)([^\?\&\=]*)",
## "%s" % referer)
## if m:
## host = m.group(2)
## dbh.insert("http", url=referer, host=host)
## parent = dbh.autoincrement()
## else:
## parent = row['inode_id']
args = dict(
inode_id=inode_id,
request_packet=p.request.get("packet_id", 0),
method=p.request.get("method", "-"),
url=url,
response_packet=p.response.get("packet_id"),
status=p.response.get("HTTP_code"),
content_type=p.response.get("content-type", "text/html"),
referrer=referer[:500],
host=host,
tld=make_tld(host),
useragent=p.request.get('user-agent', '-'),
)
if date:
args['date'] = date
dbh.insert('http', **args)
# parent = parent)
## Replicate the information about the subobjects in the
## connection_details table - this makes it easier to do
## some queries:
dbh.insert(
"connection_details",
ts_sec=stream.ts_sec,
inode_id=inode_id,
src_ip=stream.src_ip,
src_port=stream.src_port,
dest_ip=stream.dest_ip,
dest_port=stream.dest_port,
)
## handle the request's parameters:
try:
self.handle_parameters(p.request, inode_id)
except (KeyError, TypeError):
pass
## Only scan the new file using the scanner train if its
## size of bigger than 0:
if size > 0:
self.scan_as_file(new_inode, factories)
def process_stream(self, stream, factories):
""" We look for HTTP requests to identify the stream. This
allows us to processes HTTP connections on unusual ports. This
situation might arise if HTTP proxies are used for example.
"""
if stream.reverse:
combined_inode = "I%s|S%s/%s" % (stream.fd.name, stream.inode_id, stream.reverse)
try:
fd = self.fsfd.open(inode=combined_inode)
## If we cant open the combined stream, we quit (This could
## happen if we are trying to operate on a combined stream
## already
except IOError: return
else:
fd = stream
p=HTTP(fd,self.fsfd)
## Check that this is really HTTP
if not p.identify():
return
pyflaglog.log(pyflaglog.DEBUG,"Openning %s for HTTP" % combined_inode)
## Iterate over all the messages in this connection
for f in p.parse():
if not f: continue
offset, size = f
## Create the VFS node:
new_inode="%s|H%s:%s" % (combined_inode,offset,size)
try:
if 'chunked' in p.response['transfer-encoding']:
new_inode += "|c0"
except KeyError:
pass
try:
if 'gzip' in p.response['content-encoding']:
new_inode += "|G1"
except KeyError:
pass
try:
if 'deflate' in p.response['content-encoding']:
new_inode += "|d1"
except KeyError:
pass
## stream.ts_sec is already formatted in DB format
## need to convert back to utc/gmt as paths are UTC
timestamp = fd.get_packet_ts(offset)
ds_timestamp = Time.convert(timestamp, case=self.case, evidence_tz="UTC")
try:
date_str = ds_timestamp.split(" ")[0]
except:
date_str = stream.ts_sec.split(" ")[0]
path,inode,inode_id=self.fsfd.lookup(inode=combined_inode)
## Try to put the HTTP inodes at the mount point. FIXME:
## This should not be needed when a http stats viewer is
## written.
path=posixpath.normpath(path+"/../../../../../")
inode_id = self.fsfd.VFSCreate(None,new_inode,
"%s/HTTP/%s/%s" % (path,date_str,
escape(p.request['url'])),
mtime=timestamp, size=size
)
## Update the inode again:
#new_inode = new_inode % inode_id
## This updates the inode table with the new inode
#self.fsfd.VFSCreate(None,new_inode,
# None, update_only = True,
# inode_id = inode_id
# )
## Store information about this request in the
## http table:
host = p.request.get("host",IP2str(stream.dest_ip))
url = HTML.url_unquote(p.request.get("url"))
try:
date = p.response["date"]
date = Time.parse(date, case=self.case, evidence_tz=None)
except (KeyError,ValueError):
date = 0
## Two forms for the referrer:
referer = p.request.get('referer', p.request.get('referrer',''))
if not url.startswith("http://") and not url.startswith("ftp://"):
url = "http://%s%s" % (host, url)
## Not sure if we really care about this?
## Find referred page:
## parent = 0
dbh = DB.DBO(self.case)
## if referer:
## dbh.execute("select inode_id from http where url=%r order by inode_id desc limit 1", referer)
## row = dbh.fetch()
## ## If there is no referrer we just make a psuedo entry
## if not row:
## ## Find out the host
## m=re.match("(http://|ftp://)([^/]+)([^\?\&\=]*)",
## "%s" % referer)
## if m:
## host = m.group(2)
## dbh.insert("http", url=referer, host=host)
## parent = dbh.autoincrement()
## else:
## parent = row['inode_id']
args = dict(inode_id = inode_id,
request_packet = p.request.get("packet_id",0),
method = p.request.get("method","-"),
url = url,
response_packet= p.response.get("packet_id"),
status = p.response.get("HTTP_code"),
content_type = p.response.get("content-type","text/html"),
referrer = referer[:500],
host = host,
tld = make_tld(host),
useragent = p.request.get('user-agent', '-'),
)
if date:
args['date'] = date
dbh.insert('http', **args)
# parent = parent)
## Replicate the information about the subobjects in the
## connection_details table - this makes it easier to do
## some queries:
dbh.insert("connection_details",
ts_sec = stream.ts_sec,
inode_id = inode_id,
src_ip = stream.src_ip,
src_port = stream.src_port,
dest_ip = stream.dest_ip,
dest_port = stream.dest_port,
)
## handle the request's parameters:
try:
self.handle_parameters(p.request, inode_id)
except (KeyError, TypeError):
pass
## Only scan the new file using the scanner train if its
## size of bigger than 0:
if size>0:
self.scan_as_file(new_inode, factories)
def MSG(self, items, fd, scanners):
""" Sends message to members of the current session
There are two types of messages that may be sent:
1) A message from the client to the message server.
This does not contain the nick of the client, but does contain a
transaction ID. This message is sent to all users in the
current session.
2) A message from the Switchboard server to the client contains
the nick of the sender.
These two commands are totally different.
1.
MSG 1532 U 92
MIME-Version: 1.0
Content-Type: text/x-msmsgscontrol
TypingUser: [email protected]
Format is: MSG <Transaction ID> <Type of ACK required> <length of message in bytes>
Transaction ID is used to correlate server responses to client requests.
2.
MSG [email protected] I%20am%20so%20great 102
MIME-Version: 1.0
Content-Type: text/x-msmsgscontrol
TypingUser: [email protected]
Format is: MSG <Nick> <URL encoded displayname> <length of message in bytes>
"""
length = int(items[-1])
start = fd.tell()
end = start + length
if "@" in items[1]:
## Its type 2 (see above)
sender_name = "(%s)" % HTML.url_unquote(items[2])
else:
## Its type 1
sender_name = ''
sender = fd.client_id
ct = ''
while 1:
line = fd.readline().strip()
if not line: break
header, value = line.split(":",1)
header = header.lower()
if header == 'typinguser':
fd.client_id = fd.reverse.dest_id = value.strip()
elif header == 'content-type':
ct = value
## Update the start to be start start of this line
start = fd.tell()
fd.seek(end - start, 1)
## We only care about text messages here
if end > start and 'text/plain' in ct:
## Lets find out the timestamp of this point
CacheManager.urn_insert_to_table(fd.urn, "msn_session",
dict(session_id = self.session_id,
_time = "from_unixtime(%s)" % fd.current_packet.ts_sec,
offset = start,
length = end - start,
sender = fd.client_id,
recipient = fd.dest_id,
type = 'MESSAGE',
))
def MSG(self, items, fd, scanners):
""" Sends message to members of the current session
There are two types of messages that may be sent:
1) A message from the client to the message server.
This does not contain the nick of the client, but does contain a
transaction ID. This message is sent to all users in the
current session.
2) A message from the Switchboard server to the client contains
the nick of the sender.
These two commands are totally different.
1.
MSG 1532 U 92
MIME-Version: 1.0
Content-Type: text/x-msmsgscontrol
TypingUser: [email protected]
Format is: MSG <Transaction ID> <Type of ACK required> <length of message in bytes>
Transaction ID is used to correlate server responses to client requests.
2.
MSG [email protected] I%20am%20so%20great 102
MIME-Version: 1.0
Content-Type: text/x-msmsgscontrol
TypingUser: [email protected]
Format is: MSG <Nick> <URL encoded displayname> <length of message in bytes>
"""
length = int(items[-1])
start = fd.tell()
end = start + length
if "@" in items[1]:
## Its type 2 (see above)
sender_name = "(%s)" % HTML.url_unquote(items[2])
else:
## Its type 1
sender_name = ''
sender = fd.client_id
ct = ''
while 1:
line = fd.readline().strip()
if not line: break
header, value = line.split(":", 1)
header = header.lower()
if header == 'typinguser':
fd.client_id = fd.reverse.dest_id = value.strip()
elif header == 'content-type':
ct = value
## Update the start to be start start of this line
start = fd.tell()
fd.seek(end - start, 1)
## We only care about text messages here
if end > start and 'text/plain' in ct:
## Lets find out the timestamp of this point
CacheManager.urn_insert_to_table(
fd.urn, "msn_session",
dict(
session_id=self.session_id,
_time="from_unixtime(%s)" % fd.current_packet.ts_sec,
offset=start,
length=end - start,
sender=fd.client_id,
recipient=fd.dest_id,
type='MESSAGE',
))
