def makeRequest(self, request):
"""Connect to the remote HTTP server and send request"""
self.tcpclient = None
self.httpparser = None
port = intval(request.requestobject.get("uri-port", ""))
if port == None:
port = 80
self.tcpclient = self.connectionclass(
request.requestobject["uri-server"], port)
self.httpparser = HTTPParser(mode="response")
self.link((self, "_tcpoutbox"), (self.tcpclient, "inbox"))
self.link((self, "_tcpsignal"), (self.tcpclient, "control"))
self.link((self.tcpclient, "signal"), (self, "_tcpcontrol"))
self.link((self.tcpclient, "outbox"),
(self.httpparser,
"inbox")) #incoming TCP data -> HTTPParser directly
self.link((self, "_parsersignal"), (self.httpparser, "control"))
self.link((self.httpparser, "outbox"), (self, "_parserinbox"))
self.link((self.httpparser, "signal"), (self, "_parsercontrol"))
self.addChildren(self.tcpclient, self.httpparser)
self.tcpclient.activate()
self.httpparser.activate()
self.response = ""
if isinstance(request.requestobject["request"], str):
self.send(request.requestobject["request"], "_tcpoutbox")
else:
for part in request.requestobject["request"]:
self.send(part, "_tcpoutbox")
def initialiseComponent(self):
"""Create an HTTPParser component to convert the requests we receive
into a more convenient form, and a HTTPRequestHandler component to
sort out the correct response to requests received."""
self.mimehandler = HTTPParser()
self.httphandler = HTTPRequestHandler(createRequestHandler)
#self.httphandler.filereader = TriggeredFileReader()
self.link((self, "mime-control"), (self.mimehandler, "control"))
self.link((self.mimehandler, "signal"), (self, "mime-signal"))
self.link((self.mimehandler, "outbox"), (self.httphandler, "inbox"))
self.link((self, "http-control"), (self.httphandler, "control"))
self.link((self.httphandler, "signal"), (self, "http-signal"))
self.addChildren(self.mimehandler,
self.httphandler) #self.httphandler.filereader)
self.httphandler.activate()
self.mimehandler.activate()
#self.httphandler.filereader.activate()
self.link((self.httphandler, "outbox"), (self, "outbox"),
passthrough=2)
self.link((self, "inbox"), (self.mimehandler, "inbox"), passthrough=1)
def handle_packets(self, packet):
global STOP
if STOP:
print("stop sniff")
return
nowTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
global PACKET_NUM
PACKET_NUM += 1
lenth = len(packet)
ip_forward(packet)
if int(packet.getlayer(Ether).type) == 34525:
self.ipv6_count += 1
proto = 'IPv6'
src = str(packet.getlayer(IPv6).src)
dst = str(packet.getlayer(IPv6).dst)
info = str(packet.summary())
self.newPacketCatched.emit(str(PACKET_NUM), nowTime, src, dst,
proto, str(lenth), info)
PACKETS.append(packet)
elif int(packet.getlayer(Ether).type) == 2048:
self.ipv4_count += 1
src = str(packet.getlayer(IP).src)
dst = str(packet.getlayer(IP).dst)
info = str(packet.summary())
proto = ''
if int(packet.getlayer(IP).proto) == 6:
self.tcp_count += 1
proto = 'TCP'
if packet.haslayer(TCP):
if HTTPParser.isHTTP(packet):
self.http_count += 1
proto = 'HTTP'
info = HTTPParser.generateInfo(packet)
elif int(packet.getlayer(IP).proto) == 17:
proto = 'UDP'
self.udp_count += 1
elif int(packet.getlayer(IP).proto) == 1:
proto = 'ICMP'
self.icmp_count += 1
self.newPacketCatched.emit(str(PACKET_NUM), nowTime, src, dst,
proto, str(lenth), info)
PACKETS.append(packet)
elif int(packet.getlayer(Ether).type) == 2054:
proto = 'ARP'
self.arp_count += 1
src = str(packet.getlayer(ARP).psrc)
dst = str(packet.getlayer(ARP).pdst)
info = str(packet.summary())
#self.packet_table.row_append(src, dst, proto, info)
self.newPacketCatched.emit(str(PACKET_NUM), nowTime, src, dst,
proto, str(lenth), info)
PACKETS.append(packet)
self.updateCount.emit(PACKET_NUM, self.ipv4_count, self.ipv6_count,
self.tcp_count, self.udp_count, self.arp_count,
self.http_count, self.icmp_count)
def makeRequest(self, request):
"""Connect to the remote HTTP server and send request"""
self.tcpclient = None
self.httpparser = None
port = intval(request.requestobject.get("uri-port", ""))
if port == None:
port = 80
self.tcpclient = self.connectionclass(request.requestobject["uri-server"], port)
self.httpparser = HTTPParser(mode="response")
self.link( (self, "_tcpoutbox"), (self.tcpclient, "inbox") )
self.link( (self, "_tcpsignal"), (self.tcpclient, "control") )
self.link( (self.tcpclient, "signal"), (self, "_tcpcontrol") )
self.link( (self.tcpclient, "outbox"), (self.httpparser, "inbox") ) #incoming TCP data -> HTTPParser directly
self.link( (self, "_parsersignal"), (self.httpparser, "control") )
self.link( (self.httpparser, "outbox"), (self, "_parserinbox") )
self.link( (self.httpparser, "signal"), (self, "_parsercontrol") )
self.addChildren( self.tcpclient, self.httpparser )
self.tcpclient.activate()
self.httpparser.activate()
self.response = ""
if isinstance(request.requestobject["request"], str):
self.send(request.requestobject["request"], "_tcpoutbox")
else:
for part in request.requestobject["request"]:
self.send(part, "_tcpoutbox")
def initialiseComponent(self):
"""Create an HTTPParser component to convert the requests we receive
into a more convenient form, and a HTTPRequestHandler component to
sort out the correct response to requests received."""
self.mimehandler = HTTPParser()
self.httphandler = HTTPRequestHandler(createRequestHandler)
#self.httphandler.filereader = TriggeredFileReader()
self.link( (self,"mime-control"), (self.mimehandler,"control") )
self.link( (self.mimehandler, "signal"), (self, "mime-signal") )
self.link( (self.mimehandler, "outbox"), (self.httphandler, "inbox") )
self.link( (self, "http-control"), (self.httphandler, "control") )
self.link( (self.httphandler, "signal"), (self, "http-signal") )
self.addChildren(self.mimehandler, self.httphandler) #self.httphandler.filereader)
self.httphandler.activate()
self.mimehandler.activate()
#self.httphandler.filereader.activate()
self.link((self.httphandler, "outbox"), (self, "outbox"), passthrough=2)
self.link((self, "inbox"), (self.mimehandler, "inbox"), passthrough=1)
def setPacket(self, packet):
if packet is None:
return
self.rootItem.clearChildren()
src = packet.getlayer(Ether).src
dst = packet.getlayer(Ether).dst
eth_type = str(packet.getlayer(Ether).type)
if int(packet.getlayer(Ether).type) == 34525: #IPv6
eth_type = "IPv6(0x86DD)"
elif int(packet.getlayer(Ether).type) == 2048: #IPv4
eth_type = "IPv4(0x0800)"
elif int(packet.getlayer(Ether).type) == 2054: #ARP
eth_type = "ARP(0x0800)"
EtherNet_item = TreeItem(["Ethernet II"], self.rootItem)
src_item = TreeItem(["Source: " + src], EtherNet_item)
dst_item = TreeItem(["Destination: " + dst], EtherNet_item)
type_item = TreeItem(["Type: " + eth_type], EtherNet_item)
EtherNet_item.appendChild(src_item)
EtherNet_item.appendChild(dst_item)
EtherNet_item.appendChild(type_item)
EtherNet_item.itemData = [
"Ethernet II, " + "Src: " + src + ", Dst: " + dst
]
self.rootItem.appendChild(EtherNet_item)
#-----EtherNet end----------
if int(packet.getlayer(Ether).type) == 34525:
proto = 'IPv6'
src = str(packet.getlayer(IPv6).src)
dst = str(packet.getlayer(IPv6).dst)
info = str(packet.summary())
elif int(packet.getlayer(Ether).type) == 2048:
proto = "Ethernet"
if int(packet.getlayer(IP).proto) == 6:
proto = 'TCP'
elif int(packet.getlayer(IP).proto) == 17:
proto = 'UDP'
elif int(packet.getlayer(IP).proto) == 1:
proto = 'ICMP'
src = str(packet.getlayer(IP).src)
dst = str(packet.getlayer(IP).dst)
header_len = packet.getlayer(IP).ihl
header_len_str = ".... {:04b} = Header Length: {:d} bytes({:d})".format(
header_len, header_len * 4, header_len)
total_len_str = "Total length: {:d}".format(
packet.getlayer(IP).len)
identification_str = "Identification: 0x{:04x} ({:d})".format(
packet.getlayer(IP).id,
packet.getlayer(IP).id)
# flags_str = "Flag: 0x{:04x}".format(packet.getlayer(IP).flag) TODO
ttl_str = "Time to live: {:d}".format(packet.getlayer(IP).ttl)
proto_str = "Protocol: {proto_str} ({:d})".format(
packet.getlayer(IP).proto, proto_str=proto)
chksum_str = "Header cheaksum: {:04x}".format(
packet.getlayer(IP).chksum)
info = str(packet.summary())
ipv4_item = TreeItem(
["Internet Protocal Version 4, Src:" + src + ", Dst: " + dst],
self.rootItem)
src_item = TreeItem(["Source: " + src], ipv4_item)
dst_item = TreeItem(["Destination: " + dst], ipv4_item)
version_item = TreeItem(["0100 .... = Version 4"], ipv4_item)
header_len_item = TreeItem([header_len_str], ipv4_item)
total_len_item = TreeItem([total_len_str], ipv4_item)
identification_item = TreeItem([identification_str], ipv4_item)
# flags_item = TreeItem([flags_str], ipv4_item)
ttl_item = TreeItem([ttl_str], ipv4_item)
proto_item = TreeItem([proto_str], ipv4_item)
chksum_item = TreeItem([chksum_str], ipv4_item)
ipv4_item.appendChild(version_item)
ipv4_item.appendChild(header_len_item)
ipv4_item.appendChild(total_len_item)
ipv4_item.appendChild(identification_item)
# ipv4_item.appendChild(flags_item)
ipv4_item.appendChild(ttl_item)
ipv4_item.appendChild(proto_item)
ipv4_item.appendChild(chksum_item)
ipv4_item.appendChild(src_item)
ipv4_item.appendChild(dst_item)
self.rootItem.appendChild(ipv4_item)
if int(packet.getlayer(IP).proto) == 6: #TCP
tcp_packet = packet.getlayer(TCP)
sport_str = "Source Port: {:d}".format(tcp_packet.sport)
dport_str = "Destination Port: {:d}".format(tcp_packet.dport)
dataofs_str = "Data offset: {:d}".format(tcp_packet.dataofs)
reserved_str = "Reserved: {:d}".format(tcp_packet.reserved)
sql_str = "Sequence Number: {:d}".format(tcp_packet.seq)
ack_str = "Acknowledge number: {:d}".format(tcp_packet.ack)
header_len_str = "0101 .... = Header Length: 20 bytes (5)"
window_str = "Window size value: {:d}".format(
tcp_packet.window)
chksum_str = "Checksum: 0x{:04x}".format(tcp_packet.chksum)
urgptr_str = "Urgent pointer: {:d}".format(tcp_packet.urgptr)
tcp_str = "Transmission Control Protocol, Src Port: {:d}, Dst Port: {:d}, " \
"Seq: {:d}, Ack: {:d}, Len: 1"\
.format(tcp_packet.sport, tcp_packet.dport,
tcp_packet.seq, tcp_packet.ack)
tcp_item = TreeItem([tcp_str], self.rootItem)
sport_item = TreeItem([sport_str], tcp_item)
dport_item = TreeItem([dport_str], tcp_item)
dataofs_item = TreeItem([dataofs_str], tcp_item)
reserved_item = TreeItem([reserved_str], tcp_item)
sql_item = TreeItem([sql_str], tcp_item)
ack_item = TreeItem([ack_str], tcp_item)
header_len_item = TreeItem([header_len_str], tcp_item)
window_item = TreeItem([window_str], tcp_item)
chksum_item = TreeItem([chksum_str], tcp_item)
urgptr_item = TreeItem([urgptr_str], tcp_item)
tcp_item.appendChild(sport_item)
tcp_item.appendChild(dport_item)
tcp_item.appendChild(dataofs_item)
tcp_item.appendChild(reserved_item)
tcp_item.appendChild(sql_item)
tcp_item.appendChild(ack_item)
tcp_item.appendChild(header_len_item)
tcp_item.appendChild(window_item)
tcp_item.appendChild(chksum_item)
tcp_item.appendChild(urgptr_item)
self.rootItem.appendChild(tcp_item)
#Parse HTTP
if HTTPParser.isHTTP(packet):
httpItem = TreeItem(["Hypertext Transfer Protocol"],
self.rootItem)
packet_str = packet.getlayer(Raw).load.decode(
errors='ignore')
HTTP_list = packet_str.split("\n")
for http_str in HTTP_list:
http_str = http_str.strip()
if len(http_str) > 0:
newItem = TreeItem([http_str], httpItem)
httpItem.appendChild(newItem)
self.rootItem.appendChild(httpItem)
elif int(packet.getlayer(IP).proto) == 17:
udp_packet = packet.getlayer(UDP)
udp_str = "User Datagram Protocol, Src Port: {:d}, Dst Port: {:d}"\
.format(udp_packet.sport, udp_packet.dport)
sport_str = "Source Port: {:d}".format(udp_packet.sport)
dport_str = "Destination Port: {:d}".format(udp_packet.dport)
len_str = "Length: {:d}".format(udp_packet.len)
chksum_str = "Checksum: 0x{:04x}".format(udp_packet.chksum)
udp_item = TreeItem([udp_str], self.rootItem)
sport_item = TreeItem([sport_str], udp_item)
dport_item = TreeItem([dport_str], udp_item)
len_item = TreeItem([len_str], udp_item)
chksum_item = TreeItem([chksum_str], udp_item)
udp_item.appendChild(sport_item)
udp_item.appendChild(dport_item)
udp_item.appendChild(len_item)
udp_item.appendChild(chksum_item)
self.rootItem.appendChild(udp_item)
if packet.haslayer(Raw):
raw_packet = udp_packet.load
data_str = "Data ({:d} bytes)".format(len(raw_packet))
raw_str = raw_packet.hex()
if len(raw_str) >= 100:
raw_str = raw_str[:100] + "..."
load_str = "Data: " + raw_str
data_item = TreeItem([data_str], self.rootItem)
load_item = TreeItem([load_str], data_item)
data_item.appendChild(load_item)
self.rootItem.appendChild(data_item)
elif int(packet.getlayer(IP).proto) == 1:
icmp_packet = packet.getlayer(ICMP)
icmp_str = "Internet Control Message Protocol"
icmp_type = ""
if icmp_packet.type == 0:
icmp_type = "(Echo (ping) reply)"
elif icmp_packet.type == 8:
icmp_type = "(Echo (ping) request)"
type_str = "Type: {:d} {icmp_type}".format(icmp_packet.type,
icmp_type=icmp_type)
code_str = "Code: {:d}".format(icmp_packet.code)
chksum_str = "Checksum: 0x{:04x}".format(icmp_packet.chksum)
icmp_item = TreeItem([icmp_str], self.rootItem)
type_item = TreeItem([type_str], icmp_item)
code_item = TreeItem([code_str], icmp_item)
chksum_item = TreeItem([chksum_str], icmp_item)
if (icmp_packet.id):
id_str = "Identifier : {:d} (0x{:04x})".format(
icmp_packet.id, icmp_packet.id)
id_item = TreeItem([id_str], icmp_item)
icmp_item.appendChild(id_item)
if (icmp_packet.seq):
seq_str = "Sequence number : {:d} (0x{:04x})".format(
icmp_packet.seq, icmp_packet.seq)
seq_item = TreeItem([seq_str], icmp_item)
icmp_item.appendChild(seq_item)
icmp_item.appendChild(type_item)
icmp_item.appendChild(code_item)
icmp_item.appendChild(chksum_item)
self.rootItem.appendChild(icmp_item)
elif int(packet.getlayer(Ether).type) == 2054:
arp_packet = packet.getlayer(ARP)
arp_str = "Address Resolution Protocol"
hwtype_str = "Hardware type: Ethernet ({:d})".format(
arp_packet.hwtype)
ptype_str = "Protocol type: IPv4 (0x0800)"
hwlen_str = "Hardware size: {:d}".format(arp_packet.hwlen)
plen_str = "Protocol size: {:d}".format(arp_packet.plen)
op_str = "Opcode: {op} ({:d})".format(
arp_packet.op, op="request" if arp_packet.op == 1 else "reply")
hwsrc_str = "Sender MAC address: " + arp_packet.hwsrc
psrc_str = "Sender IP address: " + str(packet.getlayer(ARP).psrc)
hwdst_str = "Target MAC address: " + arp_packet.hwdst
pdst_str = "Target IP address: " + str(packet.getlayer(ARP).pdst)
arp_item = TreeItem([arp_str], self.rootItem)
hwtype_item = TreeItem([hwtype_str], arp_item)
ptype_item = TreeItem([ptype_str], arp_item)
hwlen_item = TreeItem([hwlen_str], arp_item)
plen_item = TreeItem([plen_str], arp_item)
op_item = TreeItem([op_str], arp_item)
hwsrc_item = TreeItem([hwsrc_str], arp_item)
psrc_item = TreeItem([psrc_str], arp_item)
hwdst_item = TreeItem([hwdst_str], arp_item)
pdst_item = TreeItem([pdst_str], arp_item)
arp_item.appendChild(hwtype_item)
arp_item.appendChild(ptype_item)
arp_item.appendChild(hwlen_item)
arp_item.appendChild(plen_item)
arp_item.appendChild(op_item)
arp_item.appendChild(hwsrc_item)
arp_item.appendChild(psrc_item)
arp_item.appendChild(hwdst_item)
arp_item.appendChild(pdst_item)
self.rootItem.appendChild(arp_item)
self.modelReset.emit()
class SingleShotHTTPClient(component):
"""\
SingleShotHTTPClient() -> component that can download a file using HTTP by URL
Arguments:
- starturl -- the URL of the file to download
- [postbody] -- data to POST to that URL
- [connectionclass] -- specify a class other than TCPClient to connect with
"""
Inboxes = {
"inbox": "UNUSED",
"control": "UNUSED",
"_parserinbox": "Data from HTTP parser",
"_parsercontrol": "Signals from HTTP parser",
"_tcpcontrol": "Signals from TCP client",
}
Outboxes = {
"outbox": "Requested file",
"debug": "Output to aid debugging",
"_parsersignal": "Signals for HTTP parser",
"_tcpoutbox": "Send over TCP connection",
"_tcpsignal": "Signals shutdown of TCP connection",
"signal": "UNUSED"
}
def __init__(self, starturl, postbody="", connectionclass=TCPClient):
#print "SingleShotHTTPClient.__init__()"
super(SingleShotHTTPClient, self).__init__()
self.tcpclient = None
self.httpparser = None
self.requestqueue = []
self.starturl = starturl
self.connectionclass = connectionclass
self.postbody = postbody
#print "Start url: " + starturl
def formRequest(self, url):
"""Craft a HTTP request string for the supplied url"""
splituri = splitUri(url)
host = splituri["uri-server"]
if splituri.has_key("uri-port"):
host += ":" + splituri["uri-port"]
splituri["request"] = []
if self.postbody == "":
splituri["request"].append("GET " + splituri["raw-uri"] +
" HTTP/1.1\r\n")
else:
splituri["request"].append("POST " + splituri["raw-uri"] +
" HTTP/1.1\r\n")
splituri["request"].append("Content-Length: " +
str(len(self.postbody)) + "\r\n")
splituri["request"].append("Host: " + host + "\r\n")
splituri["request"].append(
"User-agent: Kamaelia HTTP Client 0.3 (RJL)\r\n")
splituri["request"].append(
"Connection: Keep-Alive\r\n"
) # keep-alive is a work around for lack of shutdown notification in TCPClient
splituri["request"].append("\r\n")
splituri["request"] = [string.join(
splituri["request"],
"")] # might improve performance by sending more together
if self.postbody != "":
splituri["request"].append(self.postbody)
return splituri
def makeRequest(self, request):
"""Connect to the remote HTTP server and send request"""
self.tcpclient = None
self.httpparser = None
port = intval(request.requestobject.get("uri-port", ""))
if port == None:
port = 80
self.tcpclient = self.connectionclass(
request.requestobject["uri-server"], port)
self.httpparser = HTTPParser(mode="response")
self.link((self, "_tcpoutbox"), (self.tcpclient, "inbox"))
self.link((self, "_tcpsignal"), (self.tcpclient, "control"))
self.link((self.tcpclient, "signal"), (self, "_tcpcontrol"))
self.link((self.tcpclient, "outbox"),
(self.httpparser,
"inbox")) #incoming TCP data -> HTTPParser directly
self.link((self, "_parsersignal"), (self.httpparser, "control"))
self.link((self.httpparser, "outbox"), (self, "_parserinbox"))
self.link((self.httpparser, "signal"), (self, "_parsercontrol"))
self.addChildren(self.tcpclient, self.httpparser)
self.tcpclient.activate()
self.httpparser.activate()
self.response = ""
if isinstance(request.requestobject["request"], str):
self.send(request.requestobject["request"], "_tcpoutbox")
else:
for part in request.requestobject["request"]:
self.send(part, "_tcpoutbox")
def shutdownKids(self):
"""Close TCP connection and HTTP parser"""
if self.tcpclient != None and self.httpparser != None:
self.send(producerFinished(), "_tcpsignal")
self.send(shutdown(), "_parsersignal")
self.removeChild(self.tcpclient)
self.removeChild(self.httpparser)
self.tcpclient = None
self.httpparser = None
def handleRedirect(self, header):
"""Check for a redirect response and queue the fetching the page it points to if it is such a response.
Returns true if it was a redirect page and false otherwise."""
if header["responsecode"] == "302" or header[
"responsecode"] == "303" or header["responsecode"] == "307":
# location header gives the redirect URL
newurl = header["headers"].get("location", "")
if newurl != "":
redirectedrequest = HTTPRequest(
self.formRequest(newurl),
self.currentrequest.redirectcount + 1)
self.requestqueue.append(redirectedrequest)
return True
else:
return False
# do something equivalent to what we'd do for 404
else:
return False
def main(self):
"""Main loop."""
self.requestqueue.append(
HTTPRequest(self.formRequest(self.starturl), 0))
while self.mainBody():
#print "SingleShotHTTPClient.main"
yield 1
self.send(producerFinished(self), "signal")
yield 1
return
def mainBody(self):
"""Called repeatedly by main loop. Checks inboxes and processes messages received.
Start the fetching of the new page if the current one is a redirect and has been
completely fetched."""
self.send("SingleShotHTTPClient.mainBody()", "debug")
while self.dataReady("_parserinbox"):
msg = self.recv("_parserinbox")
if isinstance(msg, ParsedHTTPHeader):
self.send(
"SingleShotHTTPClient received a ParsedHTTPHeader on _parserinbox",
"debug")
# if the page is a redirect page
if not self.handleRedirect(msg.header):
if msg.header["responsecode"] == "200":
self.send(
msg, "outbox"
) # if not redirecting then send the response on
else: #treat as not found
pass
elif isinstance(msg, ParsedHTTPBodyChunk):
self.send(
"SingleShotHTTPClient received a ParsedHTTPBodyChunk on _parserinbox",
"debug")
if len(self.requestqueue
) == 0: # if not redirecting then send the response on
self.send(msg, "outbox")
elif isinstance(msg, ParsedHTTPEnd):
self.send(
"SingleShotHTTPClient received a ParsedHTTPEnd on _parserinbox",
"debug")
if len(self.requestqueue
) == 0: # if not redirecting then send the response on
self.send(msg, "outbox")
self.shutdownKids()
return 1
while self.dataReady("_parsercontrol"):
temp = self.recv("_parsercontrol")
self.send(
"SingleShotHTTPClient received something on _parsercontrol",
"debug")
while self.dataReady("_tcpcontrol"):
msg = self.recv("_tcpcontrol")
self.send(msg, "_parsersignal")
while self.dataReady("control"):
msg = self.recv("control")
if isinstance(msg, shutdownMicroprocess) or isinstance(
msg, shutdown):
self.shutdownKids()
return 0
# if we're not currently downloading a page
if self.tcpclient == None:
# then either we've finished or we should download the next URL (if we've been redirected)
if len(self.requestqueue) > 0:
self.currentrequest = self.requestqueue.pop(0)
if self.currentrequest.redirectcount == 3: # 3 redirects is excessive, give up, we're probably in a loop anyway
return 0
else:
self.makeRequest(self.currentrequest)
else:
return 0
self.pause()
return 1
class HTTPServer(component):
"""\
HTTPServer() -> new HTTPServer component capable of handling a single connection
Arguments:
-- createRequestHandler - a function required by HTTPRequestHandler that
creates the appropriate request-handler component
for each request, see HTTPResourceGlue
"""
Inboxes = {
"inbox": "TCP data stream - receive",
"mime-signal": "Error signals from MIME handler",
"http-signal": "Error signals from the HTTP resource retriever",
"control": "Receive shutdown etc. signals"
}
Outboxes = {
"outbox": "TCP data stream - send",
"mime-control": "To MIME handler",
"http-control": "To HTTP resource retriever's signalling inbox",
"signal": "UNUSED"
}
def __init__(self, createRequestHandler):
super(HTTPServer, self).__init__()
self.createRequestHandler = createRequestHandler
def initialiseComponent(self):
"""Create an HTTPParser component to convert the requests we receive
into a more convenient form, and a HTTPRequestHandler component to
sort out the correct response to requests received."""
self.mimehandler = HTTPParser()
self.httphandler = HTTPRequestHandler(createRequestHandler)
#self.httphandler.filereader = TriggeredFileReader()
self.link((self, "mime-control"), (self.mimehandler, "control"))
self.link((self.mimehandler, "signal"), (self, "mime-signal"))
self.link((self.mimehandler, "outbox"), (self.httphandler, "inbox"))
self.link((self, "http-control"), (self.httphandler, "control"))
self.link((self.httphandler, "signal"), (self, "http-signal"))
self.addChildren(self.mimehandler,
self.httphandler) #self.httphandler.filereader)
self.httphandler.activate()
self.mimehandler.activate()
#self.httphandler.filereader.activate()
self.link((self.httphandler, "outbox"), (self, "outbox"),
passthrough=2)
self.link((self, "inbox"), (self.mimehandler, "inbox"), passthrough=1)
def main(self):
self.initialiseComponent()
loop = True
while loop:
yield 1
while self.dataReady("control"):
temp = self.recv("control")
if isinstance(temp, producerFinished):
self.send(temp, "mime-control")
elif isinstance(temp, shutdownMicroprocess) or isinstance(
temp, shutdown):
self.send(shutdown(), "mime-control")
self.send(shutdown(), "http-control")
#print "HTTPServer received shutdown"
loop = False
break
while self.dataReady("mime-signal"):
temp = self.recv("mime-signal")
if isinstance(temp, producerFinished):
pass
#we don't need to care yet - wait 'til the request handler finishes
while self.dataReady("http-signal"):
temp = self.recv("http-signal")
if isinstance(temp, producerFinished):
sig = producerFinished(self)
self.send(sig, "mime-control")
self.send(sig, "signal")
loop = False
#close the connection
self.pause()
self.closeDownComponent()
def closeDownComponent(self):
"Remove my subcomponents (HTTPParser, HTTPRequestHandler)"
for child in self.childComponents():
self.removeChild(child)
self.mimehandler = None
self.httphandler = None
class SingleShotHTTPClient(component):
"""\
SingleShotHTTPClient() -> component that can download a file using HTTP by URL
Arguments:
- starturl -- the URL of the file to download
- [postbody] -- data to POST to that URL
- [connectionclass] -- specify a class other than TCPClient to connect with
"""
Inboxes = {
"inbox" : "UNUSED",
"control" : "UNUSED",
"_parserinbox" : "Data from HTTP parser",
"_parsercontrol" : "Signals from HTTP parser",
"_tcpcontrol" : "Signals from TCP client",
}
Outboxes = {
"outbox" : "Requested file",
"debug" : "Output to aid debugging",
"_parsersignal" : "Signals for HTTP parser",
"_tcpoutbox" : "Send over TCP connection",
"_tcpsignal" : "Signals shutdown of TCP connection",
"signal" : "UNUSED"
}
def __init__(self, starturl, postbody = "", connectionclass = TCPClient):
#print "SingleShotHTTPClient.__init__()"
super(SingleShotHTTPClient, self).__init__()
self.tcpclient = None
self.httpparser = None
self.requestqueue = []
self.starturl = starturl
self.connectionclass = connectionclass
self.postbody = postbody
#print "Start url: " + starturl
def formRequest(self, url):
"""Craft a HTTP request string for the supplied url"""
splituri = splitUri(url)
host = splituri["uri-server"]
if splituri.has_key("uri-port"):
host += ":" + splituri["uri-port"]
splituri["request"] = []
if self.postbody == "":
splituri["request"].append("GET " + splituri["raw-uri"] + " HTTP/1.1\r\n")
else:
splituri["request"].append("POST " + splituri["raw-uri"] + " HTTP/1.1\r\n")
splituri["request"].append("Content-Length: " + str(len(self.postbody)) + "\r\n")
splituri["request"].append("Host: " + host + "\r\n")
splituri["request"].append("User-agent: Kamaelia HTTP Client 0.3 (RJL)\r\n")
splituri["request"].append("Connection: Keep-Alive\r\n") # keep-alive is a work around for lack of shutdown notification in TCPClient
splituri["request"].append("\r\n")
splituri["request"] = [string.join(splituri["request"], "")] # might improve performance by sending more together
if self.postbody != "":
splituri["request"].append(self.postbody)
return splituri
def makeRequest(self, request):
"""Connect to the remote HTTP server and send request"""
self.tcpclient = None
self.httpparser = None
port = intval(request.requestobject.get("uri-port", ""))
if port == None:
port = 80
self.tcpclient = self.connectionclass(request.requestobject["uri-server"], port)
self.httpparser = HTTPParser(mode="response")
self.link( (self, "_tcpoutbox"), (self.tcpclient, "inbox") )
self.link( (self, "_tcpsignal"), (self.tcpclient, "control") )
self.link( (self.tcpclient, "signal"), (self, "_tcpcontrol") )
self.link( (self.tcpclient, "outbox"), (self.httpparser, "inbox") ) #incoming TCP data -> HTTPParser directly
self.link( (self, "_parsersignal"), (self.httpparser, "control") )
self.link( (self.httpparser, "outbox"), (self, "_parserinbox") )
self.link( (self.httpparser, "signal"), (self, "_parsercontrol") )
self.addChildren( self.tcpclient, self.httpparser )
self.tcpclient.activate()
self.httpparser.activate()
self.response = ""
if isinstance(request.requestobject["request"], str):
self.send(request.requestobject["request"], "_tcpoutbox")
else:
for part in request.requestobject["request"]:
self.send(part, "_tcpoutbox")
def shutdownKids(self):
"""Close TCP connection and HTTP parser"""
if self.tcpclient != None and self.httpparser != None:
self.send(producerFinished(), "_tcpsignal")
self.send(shutdown(), "_parsersignal")
self.removeChild(self.tcpclient)
self.removeChild(self.httpparser)
self.tcpclient = None
self.httpparser = None
def handleRedirect(self, header):
"""Check for a redirect response and queue the fetching the page it points to if it is such a response.
Returns true if it was a redirect page and false otherwise."""
if header["responsecode"] == "302" or header["responsecode"] == "303" or header["responsecode"] == "307":
# location header gives the redirect URL
newurl = header["headers"].get("location", "")
if newurl != "":
redirectedrequest = HTTPRequest(self.formRequest(newurl), self.currentrequest.redirectcount + 1)
self.requestqueue.append(redirectedrequest)
return True
else:
return False
# do something equivalent to what we'd do for 404
else:
return False
def main(self):
"""Main loop."""
self.requestqueue.append(HTTPRequest(self.formRequest(self.starturl), 0))
while self.mainBody():
#print "SingleShotHTTPClient.main"
yield 1
self.send(producerFinished(self), "signal")
yield 1
return
def mainBody(self):
"""Called repeatedly by main loop. Checks inboxes and processes messages received.
Start the fetching of the new page if the current one is a redirect and has been
completely fetched."""
self.send("SingleShotHTTPClient.mainBody()", "debug")
while self.dataReady("_parserinbox"):
msg = self.recv("_parserinbox")
if isinstance(msg, ParsedHTTPHeader):
self.send("SingleShotHTTPClient received a ParsedHTTPHeader on _parserinbox", "debug")
# if the page is a redirect page
if not self.handleRedirect(msg.header):
if msg.header["responsecode"] == "200":
self.send(msg, "outbox") # if not redirecting then send the response on
else: #treat as not found
pass
elif isinstance(msg, ParsedHTTPBodyChunk):
self.send("SingleShotHTTPClient received a ParsedHTTPBodyChunk on _parserinbox", "debug")
if len(self.requestqueue) == 0: # if not redirecting then send the response on
self.send(msg, "outbox")
elif isinstance(msg, ParsedHTTPEnd):
self.send("SingleShotHTTPClient received a ParsedHTTPEnd on _parserinbox", "debug")
if len(self.requestqueue) == 0: # if not redirecting then send the response on
self.send(msg, "outbox")
self.shutdownKids()
return 1
while self.dataReady("_parsercontrol"):
temp = self.recv("_parsercontrol")
self.send("SingleShotHTTPClient received something on _parsercontrol", "debug")
while self.dataReady("_tcpcontrol"):
msg = self.recv("_tcpcontrol")
self.send(msg, "_parsersignal")
while self.dataReady("control"):
msg = self.recv("control")
if isinstance(msg, shutdownMicroprocess) or isinstance(msg, shutdown):
self.shutdownKids()
return 0
# if we're not currently downloading a page
if self.tcpclient == None:
# then either we've finished or we should download the next URL (if we've been redirected)
if len(self.requestqueue) > 0:
self.currentrequest = self.requestqueue.pop(0)
if self.currentrequest.redirectcount == 3: # 3 redirects is excessive, give up, we're probably in a loop anyway
return 0
else:
self.makeRequest(self.currentrequest)
else:
return 0
self.pause()
return 1
class HTTPServer(component):
"""\
HTTPServer() -> new HTTPServer component capable of handling a single connection
Arguments:
-- createRequestHandler - a function required by HTTPRequestHandler that
creates the appropriate request-handler component
for each request, see HTTPResourceGlue
"""
Inboxes = { "inbox" : "TCP data stream - receive",
"mime-signal" : "Error signals from MIME handler",
"http-signal" : "Error signals from the HTTP resource retriever",
"control" : "Receive shutdown etc. signals" }
Outboxes = { "outbox" : "TCP data stream - send",
"mime-control" : "To MIME handler",
"http-control" : "To HTTP resource retriever's signalling inbox",
"signal" : "UNUSED" }
def __init__(self, createRequestHandler):
super(HTTPServer, self).__init__()
self.createRequestHandler = createRequestHandler
def initialiseComponent(self):
"""Create an HTTPParser component to convert the requests we receive
into a more convenient form, and a HTTPRequestHandler component to
sort out the correct response to requests received."""
self.mimehandler = HTTPParser()
self.httphandler = HTTPRequestHandler(createRequestHandler)
#self.httphandler.filereader = TriggeredFileReader()
self.link( (self,"mime-control"), (self.mimehandler,"control") )
self.link( (self.mimehandler, "signal"), (self, "mime-signal") )
self.link( (self.mimehandler, "outbox"), (self.httphandler, "inbox") )
self.link( (self, "http-control"), (self.httphandler, "control") )
self.link( (self.httphandler, "signal"), (self, "http-signal") )
self.addChildren(self.mimehandler, self.httphandler) #self.httphandler.filereader)
self.httphandler.activate()
self.mimehandler.activate()
#self.httphandler.filereader.activate()
self.link((self.httphandler, "outbox"), (self, "outbox"), passthrough=2)
self.link((self, "inbox"), (self.mimehandler, "inbox"), passthrough=1)
def main(self):
self.initialiseComponent()
loop = True
while loop:
yield 1
while self.dataReady("control"):
temp = self.recv("control")
if isinstance(temp, producerFinished):
self.send(temp, "mime-control")
elif isinstance(temp, shutdownMicroprocess) or isinstance(temp, shutdown):
self.send(shutdown(), "mime-control")
self.send(shutdown(), "http-control")
#print "HTTPServer received shutdown"
loop = False
break
while self.dataReady("mime-signal"):
temp = self.recv("mime-signal")
if isinstance(temp, producerFinished):
pass
#we don't need to care yet - wait 'til the request handler finishes
while self.dataReady("http-signal"):
temp = self.recv("http-signal")
if isinstance(temp, producerFinished):
sig = producerFinished(self)
self.send(sig, "mime-control")
self.send(sig, "signal")
loop = False
#close the connection
self.pause()
self.closeDownComponent()
def closeDownComponent(self):
"Remove my subcomponents (HTTPParser, HTTPRequestHandler)"
for child in self.childComponents():
self.removeChild(child)
self.mimehandler = None
self.httphandler = None
