def requestFilled(self, serviceID, requestID, response):
multResponse = MultiplicationResultPB()
response.populateMessage(multResponse)
Log.message("made it to request filled with request GDP ID = " +
response.dataProductID + " and response = " +
str(multResponse.result))
global done
done = True
def requestFilled(self, serviceID, requestID, response):
testPB = PythonTestPB()
response.populateMessage(testPB)
Log.message("made it to request filled with request GDP ID = "+response.dataProductID +" and count = " + str(testPB.count))
self.reqCount = testPB.count
if self.reqCount < 5:
gdp = GravityDataProduct("ServiceRequest")
gdp.data = testPB
self.gravityNode.request("ServiceTest", gdp, self)
def testHB(gravityNode):
hbListener = TestHBListener()
gravityNode.registerHeartbeatListener("PythonTest", 100000, hbListener)
gravityNode.startHeartbeat(100000) # .1 seconds
count = 0
while count < 10 and hbListener.receivedCount < 5:
count += 1
time.sleep(.1)
if hbListener.receivedCount < 5:
Log.critical("didn't receive enough heartbeats. Expected {}, but received {}".format(5, hbListener.receivedCount))
return 1
else:
Log.message("Received {} heartbeats (needed {}, but more is OK)".format(hbListener.receivedCount, 5))
gravityNode.stopHeartbeat()
count = 0
while count < 10 and hbListener.missedCount < 5:
count += 1
time.sleep(.1)
if hbListener.missedCount < 5:
Log.critical("didn't miss enough heartbeats. Expected {}, but received {}".format(5, hbListener.missedCount))
return 1
else:
Log.message("Missed {} heartbeats (needed {}, but more is OK)".format(hbListener.missedCount, 5))
gravityNode.unregisterHeartbeatListener("PythonTest")
return 0
def request(self, serviceID, dataProduct):
Log.message("made it to my request!")
Log.message("for serviceID = "+serviceID)
operands = MultiplicationOperandsPB()
Log.message(str(type(operands)))
dataProduct.populateMessage(operands)
Log.message("have operands = "+str([operands.multiplicand_a, operands.multiplicand_b]))
multResponse = MultiplicationResultPB()
multResponse.result = operands.multiplicand_a * operands.multiplicand_b
gdp = GravityDataProduct("MultResponse")
gdp.setData(multResponse)
Log.message("returning response with result = "+str(multResponse.result))
return gdp
def createTempService():
tempGravityNode = GravityNode()
count = 0
while tempGravityNode.init("TempNode") != gravity.SUCCESS and count < 5:
Log.warning("failed to init, retrying...")
time.sleep(1)
count += 1
if count == 5:
Log.critical("Could not connect to ServiceDirectory")
return 1
testProv = TestProvider()
tempGravityNode.registerService("TempService", gravity.TCP, testProv)
time.sleep(2) # give the registration time to complete
return 0
def testPubSub(gravityNode):
gravityNode.registerDataProduct("PubTest", gravity.TCP)
mySub = MySubscriber()
gravityNode.subscribe("PubTest", mySub)
pubPB = PythonTestPB()
pubPB.count = 0
pubPB.message = ""
gdp = GravityDataProduct("PubTest")
while mySub.subCount < 5 and pubPB.count < 10:
pubPB.count += 1
gdp.setData(pubPB)
gravityNode.publish(gdp)
time.sleep(.1)
if mySub.subCount < 5:
Log.critical("Pub/Sub failed")
return 1
return 0
def testPubSub(gravityNode):
gravityNode.registerDataProduct("PubTest", gravity.TCP)
mySub = MySubscriber()
gravityNode.subscribe("PubTest", mySub)
Log.warning("ref count = {}".format(sys.getrefcount(mySub)))
if sys.getrefcount(mySub) != 2 + 1: # +1 for temp reference in call
raise AssertionError(
"Ref count didn't increment properly on subscribe")
pubPB = PythonTestPB()
pubPB.count = 0
pubPB.message = ""
gdp = GravityDataProduct("PubTest")
while mySub.subCount < 5 and pubPB.count < 10:
pubPB.count += 1
gdp.data = pubPB
gravityNode.publish(gdp)
time.sleep(.1)
if mySub.subCount < 5:
Log.critical("Pub/Sub failed")
return 1
# make sure ref count doesn't change when unsubscribe called with params we didn't use for subscribe
gravityNode.unsubscribe("PubTest", mySub, "aFilterNotUsedAbove")
if sys.getrefcount(mySub) != 2 + 1: # +1 for temp reference in call
raise AssertionError(
"Ref count didn't decrement properly on unsubscribe with different params"
)
# make sure ref count decrements when unsubscribe with same params
gravityNode.unsubscribe("PubTest", mySub)
if sys.getrefcount(mySub) != 1 + 1: # +1 for temp reference in call
raise AssertionError(
"Ref count didn't decrement properly on unsubscribe with same different params"
)
# make sure ref count doesn't dec a second time
gravityNode.unsubscribe("PubTest", mySub)
if sys.getrefcount(mySub) != 1 + 1: # +1 for temp reference in call
raise AssertionError(
"Ref count didn't decrement properly on second unsubscribe call")
return 0
def main():
gravityNode = GravityNode()
count = 0
while gravityNode.init("PythonTest") != gravity.SUCCESS and count < 5:
Log.warning("failed to init, retrying...")
time.sleep(1)
count += 1
if count == 5:
Log.critical("Could not connect to ServiceDirectory")
return 1
ret = testPubSub(gravityNode)
if ret != 0:
return ret
ret = testService(gravityNode)
if ret != 0:
return ret
ret = testHB(gravityNode)
if ret != 0:
return ret
ret = testServiceTimeout(gravityNode)
if ret != 0:
return ret
Log.message("Python tests successful!")
return 0
def testService(gravityNode):
testProv = TestProvider()
gravityNode.registerService("ServiceTest", gravity.TCP, testProv)
myReq = MyRequestHandler(gravityNode)
testPB = PythonTestPB()
testPB.count = 0
gdp = GravityDataProduct("ServiceRequest")
gdp.data = testPB
if gravityNode.request("jaskhf", gdp, 2) is not None:
Log.critical("Request to non-existing service should return None")
return 1
gravityNode.request("ServiceTest", gdp, myReq)
# test async
loopCount = 0
while myReq.reqCount < 5 and loopCount < 5:
loopCount += 1
time.sleep(1)
if myReq.reqCount < 5:
Log.critical(
"Asynchronous Service Request failed, expected {} on counter, but was {}"
.format(5, myReq.reqCount))
return 1
# test sync
testPB.count = 0
gdp.data = testPB
responsePB = PythonTestPB()
for i in range(0, 5):
responseGDP = gravityNode.request("ServiceTest", gdp)
responseGDP.populateMessage(responsePB)
if responsePB.count != testPB.count + 1:
Log.critical(
"Incorrect return value, got {} but expected {}".format(
responsePB.count, testPB.count + 1))
return 1
else:
Log.message(
"Received return value {} on synchronous request".format(
responsePB.count))
testPB.count += 1
gdp.data = testPB
return 0
def subscriptionFilled(self, dataProducts):
if self.train_state != TrainingState.COLLECTING: return # ignore
pointPB = DataPointPB()
for gdp in dataProducts:
gdp.populateMessage(pointPB)
if pointPB.name not in training_data:
training_data[pointPB.name] = []
training_data[pointPB.name].append((pointPB.timestamp, pointPB.value))
self.subs += 1
if self.subs == 100:
self.subs = 0
Log.message("Received %d of %d training samples. "%([len(v) for v in training_data.values()][0], training_size))
#Log.message("Training Subscriptions: " + str([(k,len(v)) for k,v in training_data.items()]))
has_enough_data = True
for k,v in training_data.items():
if len(v) < training_size:
has_enough_data = False
break
if has_enough_data:
Log.message("Calling trainmodel")
self.train_state = TrainingState.TRAINING
TrainModel(training_data, model_file, epochs=epochs)
self.train_state = TrainingState.TRAINED
def MissedHeartbeat(self, componentID, microsecond_to_last_heartbeat, interval_in_microseconds):
Log.message("HB Listener MissedHeartbeat called")
self.missedCount += 1
# on each parent.
def __init__(self):
GravitySubscriber.__init__(self)
def subscriptionFilled(self, dataProducts):
counterPB = BasicCounterDataProductPB()
for gdp in dataProducts:
gdp.populateMessage(counterPB)
Log.message("received counter with value = " +
str(counterPB.count))
mySub = MySubscriber()
gn = GravityNode()
while gn.init("PyPub") != gravity.SUCCESS:
Log.warning("failed to init, retrying...")
time.sleep(1)
gn.registerDataProduct("PythonGDP", gravity.TCP)
gn.subscribe("PythonGDP", mySub)
counterPB = BasicCounterDataProductPB()
gdp = GravityDataProduct("PythonGDP")
for i in range(1, 50):
counterPB.count = i
gdp.data = counterPB
gn.publish(gdp)
time.sleep(1)
gn.waitForExit()
def requestTimeout(self, serviceID, requestID):
Log.message("Service request timed out")
self.timeoutCount += 1
def subscriptionFilled(self, dataProducts):
counterPB = BasicCounterDataProductPB()
for gdp in dataProducts:
gdp.populateMessage(counterPB)
Log.message("received counter with value = " +
str(counterPB.count))
dataProduct.populateMessage(operands)
Log.message("have operands = " +
str([operands.multiplicand_a, operands.multiplicand_b]))
multResponse = MultiplicationResultPB()
multResponse.result = operands.multiplicand_a * operands.multiplicand_b
gdp = GravityDataProduct("MultResponse")
gdp.data = multResponse
Log.message("returning response with result = " +
str(multResponse.result))
return gdp
gn = GravityNode()
while gn.init("PyRequest") != gravity.SUCCESS:
Log.warning("failed to init, retrying...")
time.sleep(1)
requestorProvider = MyRequestorProvider()
gn.registerService("Multiplication", gravity.TCP, requestorProvider)
# Async request
operands = MultiplicationOperandsPB()
operands.multiplicand_a = 3
operands.multiplicand_b = 4
gdp = GravityDataProduct("MultRequest")
gdp.data = operands
gn.request("Multiplication", gdp, requestorProvider)
while not done:
time.sleep(1)
has_enough_data = True
for k,v in training_data.items():
if len(v) < training_size:
has_enough_data = False
break
if has_enough_data:
Log.message("Calling trainmodel")
self.train_state = TrainingState.TRAINING
TrainModel(training_data, model_file, epochs=epochs)
self.train_state = TrainingState.TRAINED
mySub = MySubscriber()
gn = GravityNode()
while gn.init("AnomalyDetector") != gravity.SUCCESS:
Log.warning("failed to init, retrying...")
time.sleep(1)
epochs = gn.getIntParam("training_epochs", epochs)
training_size = gn.getIntParam("training_size", training_size)
channel = gn.getStringParam("subscription_name", "channel")
model_file = gn.getStringParam("model_file", "model.json")
gn.subscribe(channel, mySub)
while mySub.train_state != TrainingState.TRAINED: time.sleep(1)
Log.message("Training Complete " + str(mySub.train_state))
gn.unsubscribe(channel, mySub)
sys.exit(0)
def subscriptionFilled(self, dataProducts):
testPB = PythonTestPB()
for gdp in dataProducts:
gdp.populateMessage(testPB)
Log.message("received counter with value = "+str(testPB.count))
self.subCount = testPB.count
def ReceivedHeartbeat(self, componentID, interval_in_microseconds):
Log.message("HB Listener ReceivedHeartbeat called")
self.receivedCount += 1
self.sample_data[pointPB.name].append(
(pointPB.timestamp, pointPB.value))
# See if we have enough data from every stream to do a run of the autoencoder
# A real example would probably need to time-align and possibly interpolate the data
min_length = reduce(
lambda a, b: min(a, len(self.sample_data[b])),
self.sample_data,
len(self.sample_data[list(self.sample_data)[0]]))
if min_length >= model.n_steps:
data.put(self.sample_data)
self.sample_data = {}
gn = GravityNode()
while gn.init("AnomalyDetector") != gravity.SUCCESS:
Log.warning("failed to init, retrying...")
time.sleep(1)
model_filename = gn.getStringParam("model_file", "model.json")
model = GravityModel(model_filename)
mySub = MySubscriber()
channel = gn.getStringParam("subscription_name", "channel")
gn.subscribe(channel, mySub)
while True:
d = data.get()
mse = model.ComputeMSE(d)
Log.message("MSE = %f" % (mse))
# notreached
#**
import time, sys
from gravity import GravityNode, GravityDataProduct, gravity, GravitySubscriber, Log
from DataPoint_pb2 import DataPointPB
from datetime import datetime
import numpy as np
name = 'Pub'
anomaly_point = None
if len(sys.argv) > 1:
anomaly_point = int(sys.argv[1])
gn = GravityNode()
while gn.init(name) != gravity.SUCCESS:
Log.warning("failed to init, retrying...")
time.sleep(1)
inc1 = gn.getFloatParam("increment1", 0.05)
inc2 = gn.getFloatParam("increment2", 0.05)
channel = gn.getStringParam("subscription_name", "channel")
gn.registerDataProduct(channel, gravity.TCP)
pointPB1 = DataPointPB()
pointPB1.name = "Pub1"
gdp1 = GravityDataProduct(channel)
pointPB2 = DataPointPB()
pointPB2.name = "Pub2"
gdp2 = GravityDataProduct(channel)