def testSubscriptionForwarding6(self):
Info("---- testSubscriptionForwarding6 ----", "TestEventRouterHTTP")
evmatch = makeEvent(evtype="R1Events/ev1", source="R1Source/src1")
evdrop = makeEvent(evtype="R1Events/ev2", source="R1Source/src2")
self.doSubscriptionForwardingR2R1(None, "R1Source/src1", evmatch,
evdrop)
return
def testSubscriptionForwarding11(self):
Info("---- testSubscriptionForwarding11 ----", "TestEventRouterHTTP")
evmatch = makeEvent(evtype="R3Events1/ev1", source="R3Source1/src1")
evdrop = makeEvent(evtype="R3Events1/ev2", source="R3Source1/src2")
self.doSubscriptionForwardingR2R3("R3Events1/ev1", "R3Source1/src1",
evmatch, evdrop)
return
def testSubscriptionForwarding3(self):
Info("---- testSubscriptionForwarding3 ----",
"EventLib.TestEventHTTPClient")
evmatch = makeEvent(evtype="R3Events1/ev1", source="R3Source1/src1")
evdrop = makeEvent(evtype="R3Events1/ev2", source="R3Source1/src2")
self.doSubscriptionForwardingR2R3("R3Events1/ev1", "R3Source1/src1",
evmatch, evdrop)
return
def testTestEventHTTPClientIntro(self):
Info(
"---- testTestEventHTTPClientIntro: TestEventHTTPClient takes about 100s to run",
"EventLib.TestEventHTTPClient")
print(
"\n---- TestEventHTTPClient requires TestEventHTTPServer to be running"
+ "\n and takes about 30s to run")
return
def testSubscriptionForwarding2(self):
Info("---- testSubscriptionForwarding2 ----",
"TestEventHTTPClientServer")
evmatch = makeEvent(evtype="R3Events/ev1", source="R3Source/src1")
evdrop = makeEvent(evtype="R3Events/ev2", source="R3Source/src2")
self.doSubscriptionForwardingR2R3(None, "R3Source/src1", evmatch,
evdrop)
return
def testSubscriptionForwarding1(self):
Info("---- testSubscriptionForwarding1 ----------",
"TestEventRouterHTTP")
evmatch = makeEvent(evtype="R2Events/ev1", source="R2Source/src1")
evdrop = makeEvent(evtype="R2Events/ev2", source="R2Source/src2")
self.doSubscriptionForwardingR1R2("R2Events/ev1", None, evmatch,
evdrop)
return
def testSubscriptionForwarding12(self):
Info("---- testSubscriptionForwarding12 ----", "TestEventRouterHTTP")
evmatch = makeEvent(evtype="RREvents3/ev1", source="RRSource3/src1")
evdrop = makeEvent(evtype="RREvents3/ev2", source="RRSource3/src2")
self.doSubscriptionForwardingR2R3("RREvents3/ev1",
"RRSource3/src1",
evmatch,
evdrop,
r1fwd=1)
return
def testSubscriptionForwarding4(self):
Info("---- testSubscriptionForwarding4 ----",
"EventLib.TestEventHTTPClient")
evmatch = makeEvent(evtype="RREvents3/ev1", source="RRSource3/src1")
evdrop = makeEvent(evtype="RREvents3/ev2", source="RRSource3/src2")
self.doSubscriptionForwardingR2R3("RREvents3/ev1",
"RRSource3/src1",
evmatch,
evdrop,
r1fwd=1)
return
def handleEvent(self, event):
"""
Handle an incoming event. If defined, the handler function is called with
this event handler object and the event itself as arguments.
Exceptions are logged then ignored.
"""
sts = makeDeferred(StatusVal.OK)
if self._handleEvent:
try:
sts = self._handleEvent(self, event)
except Exception, ex:
# if any handler throws an exception, log and continue.
Info("Exception %s" % (ex), "EventLib.EventHandler")
def do_GET(self):
"""
Wait for event to send to client, then return it to the client.
"""
[typ, env] = self.getRelay().getQueuedItem()
Info("%s do_GET [%s,%s]: " % (self.getRelay().getUri(), typ, env),
"EventLib.EventRelayHTTPS")
if typ == "closedown":
self.send_error(503, "Request aborted - closing down")
return
if typ == "idle":
data = makeIdleData()
elif typ == "forward":
data = makeEnvelopeData(env)
else:
raise ValueError, "unexpected message type: " + str(typ)
self.send_response(200, "OK " + typ)
self.send_header('Content-type', 'application/octet-stream')
self.send_header('Content-length', str(len(data)))
self.end_headers()
# Now write data
Info("%s do_GET data: %s" % (self.getRelay().getUri(), data),
"EventLib.EventRelayHTTPS")
self.wfile.write(data)
def do_POST(self):
"""
The client is forwarding an event or subscription request:
it needs to be processed for the underlying local event router.
"""
router = self.getRelay()
msgbody = ""
l = int(self.headers.get("content-length", "0"))
if l:
msgbody = self.rfile.read(l)
Info("%s do_POST: '%s'" % (self.getRelay().getUri(), msgbody),
"EventLib.EventRelayHTTPS")
# Parse message and act accordingly
msgdata = parseMessageData(msgbody)
if msgdata == None:
self.send_error(400, "Request body malformed")
return
if msgdata[0] == "forward":
# msgdata = ["forward", [['R1', 'R2', 'R3'], 'ev:typ', 'ev:src', 'payload']]
event = makeEvent(evtype=msgdata[1][1],
source=msgdata[1][2],
payload=msgdata[1][3])
env = constructEnvelope(msgdata[1][0], event)
self.getRelay().forward(event, env)
elif msgdata[0] == "idle":
self.send_response(200, "OK idle")
self.end_headers()
return
elif msgdata[0] == "closedown":
self.send_response(200, "OK closedown")
self.end_headers()
return
else:
self.send_error(400,
"Request body unrecognized option: " + msgdata[0])
return
# Complete request with success response
self.send_response(200, "OK")
self.end_headers()
def testTestEventHTTPClientServerDone(self):
Info("---- testTestEventHTTPClientServerDone ----",
"TestEventHTTPClientServer")
print "\n---- TestEventHTTPClientServer done."
return
def processEvent(self):
"""
This function is the HTTP client worker thread.
"""
# Note: break out of event dispatch loop when closedown event is received
# and closing flag is set. This is to prevent DoS attack by faked closedown
# event type, and to ensure that prior events received are all processed.
delay_on_error_min = 0.125 # Back off retry interval on error..
delay_on_error_max = 20.0 # ..
delay_on_error = delay_on_error_min # ..
while True:
if delay_on_error < delay_on_error_max:
delay_on_error *= 2
try:
if not self._queue.empty():
Trace("%s queue.get ..." % (self.getUri()),
"EventLib.EventRelayHTTPC")
###msgbody = self._queue.get()
###Trace("%s get msgbody: %s"%(self.getUri(),msgbody), "EventLib.EventRelayHTTPC")
###self._event.set()
msgbody = self.getQueuedItem()
[typ, env] = msgbody
if typ == "closedown":
if self._closing: break
else:
# process request as an HTTP POST request
data = makeEnvelopeData(env)
headers = {
"Content-type": "text/plain",
"Accept": "text/plain",
"Content-length": str(len(data))
}
self._httpcon.request("POST", "/request_path_ignored",
data, headers)
response = self._httpcon.getresponse()
delay_on_error = delay_on_error_min
else:
# Nothing in queue:
# issue a GET for incoming events
Trace("%s HTTP get ..." % (self.getUri()),
"EventLib.EventRelayHTTPC")
headers = {"Accept": "text/plain"}
self._httpcon.request("GET", "/request_path_ignored", None,
headers)
response = self._httpcon.getresponse()
if response.status == 200:
delay_on_error = delay_on_error_min
msgbody = response.read()
Trace("%s get msgbody: %s" % (self.getUri(), msgbody),
"EventLib.EventRelayHTTPC")
# Parse message and act accordingly
msgdata = parseMessageData(msgbody)
Trace(
"%s get msgdata: %s" %
(self.getUri(), str(msgdata)),
"EventLib.EventRelayHTTPC")
if msgdata == None:
#TODO: Log "Request body malformed"
pass
elif msgdata[0] == "forward":
# msgdata = ["forward", [['R1', 'R2', 'R3'], 'ev:typ', 'ev:src', 'payload']]
event = makeEvent(evtype=msgdata[1][1],
source=msgdata[1][2],
payload=msgdata[1][3])
env = constructEnvelope(msgdata[1][0], event)
self.forward(event, env)
elif msgdata[0] == "idle":
# Idle response gives client a chance to send if anything is queued
pass
else:
#TODO: handle closedown message?
Warn(
"%s Request body unrecognized option: %s" %
(self.getUri(), msgdata[0]), "EventRelayHTTPC")
pass
elif response.status == 503:
Trace(
"%s processEvent error response: %u, %s" %
(self.getUri(), response.status, response.reason),
"EventLib.EventRelayHTTPC")
# Remote end closed down
break
else:
# TODO: (log error response)
Warn(
"%s processEvent error response: %u, %s" %
(self.getUri(), response.status, response.reason),
"EventLib.EventRelayHTTPC")
time.sleep(delay_on_error)
except httplib.BadStatusLine, e:
# This can happen at closedown
Info(
"%s processEvent bad response: %s" %
(self.getUri(), str(e)), "EventLib.EventRelayHTTPC")
time.sleep(delay_on_error)
except httplib.CannotSendRequest, e:
# This can happen at closedown
Info("%s Cannot send request: %s" % (self.getUri(), str(e)),
"EventLib.EventRelayHTTPC")
time.sleep(delay_on_error)
def testTestEventRouterHTTPIntro(self):
Info(
"---- testTestEventRouterHTTPIntro: TestEventRouterHTTP component tests take about 100s to run",
"TestEventRouterHTTP")
print "\n---- TestEventRouterHTTP component tests take about 100s to run"
return
def log_message(self, format, *args):
"""
Override default request logging
"""
Info(self.getRelay().getUri() + " " + (format % args),
"EventLib.EventRelayHTTPS-log")
def testTestEventHTTPClientDone(self):
Info("---- testTestEventHTTPClientDone ----",
"EventLib.TestEventHTTPClient")
print "\n---- TestEventHTTPClient done."
return
def testTestEventRouterHTTPDone(self):
Info(
"---- testTestEventRouterHTTPDone TestEventRouterHTTP component tests done ----",
"TestEventRouterHTTP")
print "\n---- TestEventRouterHTTP component tests done"
return
class EventRelayHTTPC(EventAgent):
"""
Implements an HTTP client event router that runs as a separate thread until
explicitly closed, which runs in tandem with a simple event router and
provides a tiny subset of the event router interface (receive).
The HTTP connection operates as a half duplex channel for sending and
receiving events, with the direction of flow being controlled by the
client: a GET request is implicitly a request for an event to be delivered
and blocks until an event is available, the request timeout period expires,
or the client cancels the request; a POST request supplies an event to be
delivered and/or forwarded.
Incoming events are queued for the client process, and are handled by the
HTTP client running in its separate thread.
"""
def __init__(self, router, uri=None, host='', port=8082):
"""
Initialize a new HTTP client event passing object
An HTTP client is associated with an existing event router, and
sends all messages received from that router to the HTTP connection,
and forwards all messages received from the HTTP connection to the
router.
Interaction with the indicated EventRouter object takes place primarily
through the 'receive' methods of this class and the supplied router.
Because messages received from HTTP are sent onwards using the normal
forwarding mechanisms, this class must perform loop-detection to stop
events being bounced back to the HTTP connection.
"""
super(EventRelayHTTPC, self).__init__(uri)
self._router = router
self._queue = Queue()
self._event = threading.Event()
self._closing = False
# Have 'router' send all subscriptions events to this object
router.routeEventFrom(None, None, self)
router.doSubscribeRequest(self, -1, None, None)
# Create HTTP "connection", and start thread to respond to new events from it.
self._httpcon = httplib.HTTPConnection(host=host, port=port)
self._thread = threading.Thread(name=uri, target=self.processEvent)
self._thread.start()
return
def receive(self, fromrouter, envelope):
"""
This function receives messages from the associated router and queues
them for transmission on the HTTP interface.
NOTE: receive and forward here perform loop-check for outgoing events,
and add the extra envelope hop for incoming. The sole purpose of this
loop-check is to prevent incoming HTTP events from being sent out again.
"""
event = envelope.unWrap(self.getUri())
if event:
Trace("%s receive %s from %s" % (self.getUri(), event, fromrouter),
"EventLib.EventRelayHTTPC")
return self.queueItem(["forward", envelope])
return makeDeferred(StatusVal.OK)
def forward(self, event, env):
"""
Internal function to process event received from HTTP connection:
add new hop to envelope and pass it straight on to the associated router object.
"""
Trace("%s forward %s" % (self.getUri(), event),
"EventLib.EventRelayHTTPC")
return self._router.receive(self, env.nextHop(self.getUri()))
def close(self):
"""
Shut down the event router thread
"""
Trace("%s close" % (self.getUri()), "EventLib.EventRelayHTTPC")
self._httpcon.close()
self._closing = True
self._event.set()
self._queue.put(["closedown", []])
self._thread.join()
Trace("%s closed" % (self.getUri()), "EventLib.EventRelayHTTPC")
return
def queueItem(self, item):
"""
Add item to the queue, and return a deferred object that fires when an item is removed
(or the queue is empty).
"""
Trace("%s queueItem (%s)" % (self.getUri(), item),
"EventLib.EventRelayHTTPC")
if not self._closing:
self._queue.put(item)
return makeQueueDeferred(StatusVal.OK, self._queue, self._event)
return makeDeferred(StatusVal.OK)
def getQueuedItem(self):
"""
Wait for an item to be queued, then return it.
"""
Trace("%s getQueuedItem ..." % (self.getUri()),
context="EventLib.EventRelayHTTPC")
item = self._queue.get()
Trace("%s getQueuedItem (%s)" % (self.getUri(), item),
context="EventLib.EventRelayHTTPC")
self._event.set()
return item
# --- HTTP client worker thread function ---
def processEvent(self):
"""
This function is the HTTP client worker thread.
"""
# Note: break out of event dispatch loop when closedown event is received
# and closing flag is set. This is to prevent DoS attack by faked closedown
# event type, and to ensure that prior events received are all processed.
delay_on_error_min = 0.125 # Back off retry interval on error..
delay_on_error_max = 20.0 # ..
delay_on_error = delay_on_error_min # ..
while True:
if delay_on_error < delay_on_error_max:
delay_on_error *= 2
try:
if not self._queue.empty():
Trace("%s queue.get ..." % (self.getUri()),
"EventLib.EventRelayHTTPC")
###msgbody = self._queue.get()
###Trace("%s get msgbody: %s"%(self.getUri(),msgbody), "EventLib.EventRelayHTTPC")
###self._event.set()
msgbody = self.getQueuedItem()
[typ, env] = msgbody
if typ == "closedown":
if self._closing: break
else:
# process request as an HTTP POST request
data = makeEnvelopeData(env)
headers = {
"Content-type": "text/plain",
"Accept": "text/plain",
"Content-length": str(len(data))
}
self._httpcon.request("POST", "/request_path_ignored",
data, headers)
response = self._httpcon.getresponse()
delay_on_error = delay_on_error_min
else:
# Nothing in queue:
# issue a GET for incoming events
Trace("%s HTTP get ..." % (self.getUri()),
"EventLib.EventRelayHTTPC")
headers = {"Accept": "text/plain"}
self._httpcon.request("GET", "/request_path_ignored", None,
headers)
response = self._httpcon.getresponse()
if response.status == 200:
delay_on_error = delay_on_error_min
msgbody = response.read()
Trace("%s get msgbody: %s" % (self.getUri(), msgbody),
"EventLib.EventRelayHTTPC")
# Parse message and act accordingly
msgdata = parseMessageData(msgbody)
Trace(
"%s get msgdata: %s" %
(self.getUri(), str(msgdata)),
"EventLib.EventRelayHTTPC")
if msgdata == None:
#TODO: Log "Request body malformed"
pass
elif msgdata[0] == "forward":
# msgdata = ["forward", [['R1', 'R2', 'R3'], 'ev:typ', 'ev:src', 'payload']]
event = makeEvent(evtype=msgdata[1][1],
source=msgdata[1][2],
payload=msgdata[1][3])
env = constructEnvelope(msgdata[1][0], event)
self.forward(event, env)
elif msgdata[0] == "idle":
# Idle response gives client a chance to send if anything is queued
pass
else:
#TODO: handle closedown message?
Warn(
"%s Request body unrecognized option: %s" %
(self.getUri(), msgdata[0]), "EventRelayHTTPC")
pass
elif response.status == 503:
Trace(
"%s processEvent error response: %u, %s" %
(self.getUri(), response.status, response.reason),
"EventLib.EventRelayHTTPC")
# Remote end closed down
break
else:
# TODO: (log error response)
Warn(
"%s processEvent error response: %u, %s" %
(self.getUri(), response.status, response.reason),
"EventLib.EventRelayHTTPC")
time.sleep(delay_on_error)
except httplib.BadStatusLine, e:
# This can happen at closedown
Info(
"%s processEvent bad response: %s" %
(self.getUri(), str(e)), "EventLib.EventRelayHTTPC")
time.sleep(delay_on_error)
except httplib.CannotSendRequest, e:
# This can happen at closedown
Info("%s Cannot send request: %s" % (self.getUri(), str(e)),
"EventLib.EventRelayHTTPC")
time.sleep(delay_on_error)
except httplib.ResponseNotReady, e:
# This can happen at startup and sometimes other times:
# maybe multiple requests on a single HTTP connection object?
Info("%s Response not ready: (%s)" % (self.getUri(), str(e)),
"EventLib.EventRelayHTTPC")
time.sleep(delay_on_error)
def testTestEventHTTPClientServerIntro(self):
Info(
"---- testTestEventHTTPClientServerIntro: TestEventHTTPClientServer takes about 100s to run",
"TestEventHTTPClientServer")
print "\n---- TestEventHTTPClientServer takes about 60s to run"
return