def test_serializePolicy(self):
qGraph = jsapi.QueryGraph()
local_cube = qGraph.add_cube("results")
local_cube.add_dim("state", Element.STRING, 0)
local_cube.add_dim("time", Element.TIME, 1)
local_cube.add_agg("count", jsapi.Cube.AggType.COUNT, 2)
src = jsapi.RandSource(qGraph, 1, 2)
sub = jsapi.TimeSubscriber(qGraph, {}, 1000,
"-count") #pull every second
sample = jsapi.VariableSampling(qGraph)
eval_op = jsapi.RandEval(qGraph)
qGraph.chain([src, local_cube, sub, sample, eval_op])
qGraph.add_policy([sub, sample])
try:
pb = qGraph.get_deploy_pb()
self.assertEquals(len(pb.alter[0].congest_policies), 1)
oid = pb.alter[0].congest_policies[0].op[0].task
self.assertEquals(oid, sub.id)
# print str(pb.alter)
except SchemaError as ex:
self.fail("should not throw, but got " + str(ex))
def test_cubeInsert(self):
qGraph = jsapi.QueryGraph()
local_cube = qGraph.add_cube("results")
local_cube.add_dim("state", Element.STRING, 0)
local_cube.add_dim("time", Element.TIME, 1)
local_cube.add_agg("count", jsapi.Cube.AggType.COUNT, 2)
src = jsapi.RandSource(qGraph, 1, 2)
qGraph.connect(src, local_cube)
try:
qGraph.validate_schemas()
except SchemaError as ex:
self.fail("should not throw, but got " + str(ex))
qGraph.remove(src)
#add a mismatched edge, string versus string,time
reader = jsapi.FileRead(qGraph, "file name")
qGraph.connect(reader, local_cube)
e_map = qGraph.forward_edge_map()
self.assertEquals(len(e_map), 1)
try:
qGraph.validate_schemas()
except SchemaError as ex:
print "got expected err:", str(ex)
else:
self.fail("should throw, but didn't")
def test_randEval(self):
qGraph = jsapi.QueryGraph()
src = jsapi.RandSource(qGraph, 1, 2)
ex = jsapi.ExtendOperator(qGraph, "i", ["a count"])
eval = jsapi.RandEval(qGraph)
qGraph.connect(src, ex)
qGraph.connect(ex, eval)
try:
qGraph.validate_schemas()
except SchemaError as ex:
self.fail("Should not throw, but got: " + str(ex))
def test_cubeInsertPartial(self):
print "inserting tuple with more data than cube"
#tests insertion where the tuple has more fields than the cube
qGraph = jsapi.QueryGraph()
local_cube = qGraph.add_cube("results")
local_cube.add_dim("state", Element.STRING, 0)
src = jsapi.RandSource(qGraph, 1, 2)
qGraph.connect(src, local_cube)
try:
qGraph.validate_schemas()
except SchemaError as ex:
self.fail("should not throw, but got " + str(ex))
def test_with_partial_placement(self):
dummyNode1 = ("host", 123)
dummyNode2 = ("host2", 234)
planner = QueryPlanner({
dummyNode1: dummyNode1,
dummyNode2: dummyNode2
})
g = jsapi.QueryGraph()
evalOp = jsapi.RandEval(g)
for node, k in zip([dummyNode1, dummyNode2], range(0, 2)):
src = jsapi.RandSource(g, 1, 2)
src.set_cfg("rate", 1000)
localCube = g.add_cube("local_results_%d" % k)
localCube.add_dim("state", Element.STRING, 0)
localCube.add_dim("time", Element.TIME, 1)
localCube.add_agg("count", jsapi.Cube.AggType.COUNT, 2)
pullOp = jsapi.TimeSubscriber(g, {}, 1000)
pullOp.set_cfg("ts_field", 1)
pullOp.set_cfg("window_offset",
1000) #pull every three seconds, trailing by one
extendOp = jsapi.ExtendOperator(g, "s", ["node" + str(k)])
roundOp = jsapi.TRoundOperator(g, fld=1, round_to=5)
g.connect(src, localCube)
g.connect(localCube, pullOp)
g.connect(pullOp, extendOp)
g.connect(extendOp, roundOp)
g.connect(roundOp, evalOp)
nID = NodeID()
nID.address, nID.portno = node
src.instantiate_on(nID)
g.validate_schemas()
err = planner.take_raw_topo(g.get_deploy_pb().alter[0])
self.assertEquals(len(err), 0)
plan = planner.get_assignments(1)
pb1 = plan[dummyNode1].get_pb().alter[0]
subscribers = [x for x in pb1.toStart if "Subscriber" in x.op_typename]
self.assertEquals(len(subscribers), len(pb1.toCreate))
self.assertEquals(len(pb1.toCreate), 1)
self.assertGreater(len(pb1.toStart), 3)
self.assertLessEqual(len(pb1.toStart), 4)
def test_bad_unify(self):
qGraph = jsapi.QueryGraph()
src = jsapi.RandSource(qGraph, 1, 2)
reader = jsapi.FileRead(qGraph, "file name")
dest = jsapi.ExtendOperator(qGraph, "s", ["a string"])
qGraph.connect(reader, dest)
qGraph.connect(src, dest)
try:
qGraph.validate_schemas()
except SchemaError as ex:
self.assertTrue("match existing schema" in str(ex))
# print "got expected err:", str(ex)
else:
self.fail("should throw, but didn't")
def test_cubeFilterSubscriber(self):
qGraph = jsapi.QueryGraph()
src = jsapi.RandSource(qGraph, 1, 2)
local_cube = qGraph.add_cube("results")
local_cube.add_dim("state", Element.STRING, 0)
local_cube.add_agg("count", jsapi.Cube.AggType.COUNT, 2)
filter = jsapi.FilterSubscriber(qGraph, cube_field=2, level_in_field=0)
#out-schema from filter should be S,T, matching source
ex = jsapi.ExtendOperator(qGraph, "i", ["a count"])
eval_op = jsapi.RandEval(qGraph)
qGraph.chain([src, ex, local_cube, filter, eval_op])
reader = jsapi.FileRead(qGraph, "file name")
csv_parse = jsapi.CSVParse(qGraph, types="I", fields_to_keep="all")
qGraph.chain([reader, csv_parse, filter])
try:
qGraph.validate_schemas()
except SchemaError as ex:
self.fail("should not throw, but got " + str(ex))
def test_line_graph_with_subscriber(self):
dummyNode1 = ("host", 123)
dummyNode2 = ("host2", 234)
dummyNode3 = ("host3", 345)
planner = QueryPlanner({
dummyNode1: dummyNode1,
dummyNode2: dummyNode2,
dummyNode3: dummyNode3
})
g = jsapi.QueryGraph()
src = jsapi.RandSource(g, 1, 2)
src.set_cfg("rate", 1000)
localCube = g.add_cube("local_results")
localCube.add_dim("state", Element.STRING, 0)
localCube.add_dim("time", Element.TIME, 1)
localCube.add_agg("count", jsapi.Cube.AggType.COUNT, 2)
pullOp = jsapi.TimeSubscriber(g, {}, 1000)
pullOp.set_cfg("ts_field", 1)
pullOp.set_cfg("window_offset",
1000) #pull every three seconds, trailing by one
remoteCube = g.add_cube("remote_results")
remoteCube.add_dim("state", Element.STRING, 0)
remoteCube.add_dim("time", Element.TIME, 1)
remoteCube.add_agg("count", jsapi.Cube.AggType.COUNT, 2)
extendOp = jsapi.ExtendOperator(g, "s", ["node1"])
roundOp = jsapi.TRoundOperator(g, fld=1, round_to=5)
# The line graph topology is: src -> cube -> subscriber -> operator(s) -> cube.
g.connect(src, localCube)
g.connect(localCube, pullOp)
g.connect(pullOp, extendOp)
g.connect(extendOp, roundOp)
g.connect(roundOp, remoteCube)
node1ID = NodeID()
node1ID.address, node1ID.portno = dummyNode1
node2ID = NodeID()
node2ID.address, node2ID.portno = dummyNode2
node3ID = NodeID()
node3ID.address, node3ID.portno = dummyNode3
g.validate_schemas()
# Pin nothing: everything should be placed on one node
err = planner.take_raw_topo(g.get_deploy_pb().alter[0])
self.assertEquals(len(err), 0)
plan = planner.get_assignments(1)
self.assertEquals(len(plan), 1)
# Pin source (src): everything should be placed on the source node
src.instantiate_on(node2ID)
err = planner.take_raw_topo(g.get_deploy_pb().alter[0])
self.assertEquals(len(err), 0)
plan = planner.get_assignments(1)
self.assertEquals(len(plan), 1)
self.assertTrue(dummyNode2 in plan)
# Pin source (src) and sink (remoteCube): everything except sink should be on source node
src.instantiate_on(node2ID)
remoteCube.instantiate_on(node1ID)
err = planner.take_raw_topo(g.get_deploy_pb().alter[0])
self.assertEquals(len(err), 0)
plan = planner.get_assignments(1)
self.assertEquals(len(plan), 2)
node1Plan = plan[dummyNode1]
node2Plan = plan[dummyNode2]
self.assertEquals(len(node1Plan.cubes), 1)
self.assertTrue(node1Plan.cubes[0].name.endswith(remoteCube.name))
self.assertEquals(len(node1Plan.operators), 0)
self.assertEquals(len(node2Plan.cubes), 1)
self.assertTrue(node2Plan.cubes[0].name.endswith(localCube.name))
self.assertEquals(len(node2Plan.operators), 4)
# Pin source (src), source cube (localCube), and sink (remoteCube): regardless of where
# source and sink are placed, source cube up to (but excluding) sink should be on same node
src.instantiate_on(node2ID)
localCube.instantiate_on(node3ID)
remoteCube.instantiate_on(node1ID)
err = planner.take_raw_topo(g.get_deploy_pb().alter[0])
self.assertEquals(len(err), 0)
plan = planner.get_assignments(1)
self.assertEquals(len(plan), 3)
node3Plan = plan[dummyNode3]
self.assertEquals(len(node3Plan.cubes), 1)
self.assertTrue(node3Plan.cubes[0].name.endswith(localCube.name))
self.assertEquals(len(node3Plan.operators), 3)
# In particular, the cube subscriber should be on the same node as the cube!
pb3 = node3Plan.get_pb().alter[0]
subscribers = [x for x in pb3.toStart if "Subscriber" in x.op_typename]
self.assertEquals(len(subscribers), 1)