def test_saveoutput():
d = StatsDatabase(testing=True)
d._StatsDatabase__save_output("module1", 1, -1, "outputname", "simlabel", "label", 123.123, table_name="dataTable")
c = d.conn.cursor()
res = c.execute('SELECT * FROM dataTable').fetchall()
assert(res[0] == ("module1", 1, "outputname", "label", -1, 123.123))
d.conn.close()
def test_readcastaliaoutput_with_nodemapping():
data_list = [('ResourceManager', 'plan0', 'Consumed Energy', '', -1, 6.79813),
('Communication.Radio', 'plan0', 'RX pkt breakdown', 'Failed with NO interference', -1, 96.0),
('Communication.Radio', 'plan0', 'RX pkt breakdown', 'Failed, below sensitivity', -1, 101.0),
('Communication.Radio', 'plan0', 'RX pkt breakdown', 'Received with NO interference', -1, 131.0),
('Application', 'plan0', 'Packets received per node', '', 'plan1', 60.0),
('Application', 'plan0', 'Packets received per node', '', 'plan2', 71.0),
('Application', 'plan0', 'Application level latency, in ms', '[0,20)', 0, 0.0),
('Application', 'plan0', 'Application level latency, in ms', '[20,40)', 1, 0.0),
('Application', 'plan0', 'Application level latency, in ms', '[40,60)', 2, 0.0),
('Application', 'plan0', 'Application level latency, in ms', '[60,80)', 3, 131.0),
('Application', 'plan0', 'Application level latency, in ms', '[80,100)', 4, 0.0),
('Application', 'plan0', 'Application level latency, in ms', '[100,120)', 5, 0.0),
('Application', 'plan0', 'Application level latency, in ms', '[120,140)', 6, 0.0),
('Application', 'plan0', 'Application level latency, in ms', '[140,160)', 7, 0.0),
('Application', 'plan0', 'Application level latency, in ms', '[160,180)', 8, 0.0),
('Application', 'plan0', 'Application level latency, in ms', '[180,200)', 9, 0.0),
('Application', 'plan0', 'Application level latency, in ms', '[200,inf)', 10, 0.0),
('ResourceManager', 'plan1', 'Consumed Energy', '', -1, 6.28785),
('Communication.Radio', 'plan1', 'TXed pkts', 'TX pkts', -1, 499.0),
('ResourceManager', 'plan2', 'Consumed Energy', '', -1, 6.28569),
('Communication.Radio', 'plan2', 'TXed pkts', 'TX pkts', -1, 499.0)]
d = StatsDatabase(testing=True)
d.load_data_castalia(castalia_output_file(), node_mapping_file=node_mapping_file())
dt = d.get_datatable()
for i in data_list:
assert(i in dt)
d.conn.close()
def test_add_network_type_parameter():
db = StatsDatabase(testing=True)
db.load_data_castalia(castalia_output_file())
# column name
c_name = Column("name")
# column SUM(data) AS data
c_data = ColumnExpr("data", Operator(SUM, [ConstantExpr("data")]))
# filter --> name = "Consumed Energy"
f = Operator(EQ, [
ColumnRef(DATA_TABLE.col["name"]),
ConstantExpr("\"Consumed Energy\"")
])
# create derived table containing the SUM of all "Consumed Energy"
dt = DerivedTable("dt", [c_name, c_data], [DATA_TABLE], f,
[DATA_TABLE.col["name"]])
# generate a view based on the above derived table
create_view_for_derived(db, dt)
# network parameter
pm_node = PlotModel("parameter",
"network",
dt,
None, (dt.col["data"], ),
title="Total Consumed Energy")
# create JsonOutput object
jo = JsonOutput("simulation_results", "46345634563456")
# fetch stats from db
stats = db.execute("SELECT data FROM dt")
# add parameter to jsonOutput object
add_network_type_parameter(jo, pm_node, stats)
correct_json = """{
"node_parameter_results": [],
"node_plot_results": [],
"node_2_node_results": [],
"network_parameter_results": [
{
"value": 19.371669999999998,
"unit": "",
"name": "Total Consumed Energy"
}
],
"type": "simulation_results",
"network_plot_results": [],
"simulation_id": "46345634563456"
}"""
assert json.loads(correct_json) == json.loads(jo.get_json_string())
def test_readCSV():
data_list = [('foo', 'plan1', '12345'),
('bar', 'plan2', '54321')]
table = nsdTable("csvTable", [Column("name"), Column("node"), Column("data")], csv_data_file(), "csv", ["node"])
d = StatsDatabase(testing=True)
create_table(d, table)
d.load_data_csv(table, node_mapping_file())
dt = d.execute("SELECT * FROM csvTable")
for i in data_list:
assert(i in dt)
d.conn.close()
def model2plots(pml, jo, castalia_data=None):
"""Accepts a list of PlotModel objects and creates the corresponding plots/parameters(statistics).
"""
assert isinstance(pml, list)
assert all(isinstance(pm, PlotModel) for pm in pml)
# Collect a list of tables, ordered according to dependence
table_list = collect_tables_for_pml(pml)
# Create database
ds = StatsDatabase()
# create tables
for table in table_list:
if isinstance(table, Table) and not isinstance(table, DerivedTable):
with Context(view=table.name):
create_table(ds, table)
populate_table(ds, table, castalia_data)
# create views
for table in table_list:
if isinstance(table, DerivedTable):
with Context(view=table.name):
create_view_for_derived(ds, table)
# create plots
for pm in pml:
if pm.rel.name in ds.relations:
with Context(view=pm.rel.name, plot=pm.title):
create_plot_for_model(pm, ds, jo)
else:
fail(
"View %s does not exist in database, some error occurred during its generation"
% pm.rel.name)
def test_createdatabase():
d = StatsDatabase(testing=True)
assert(d.conn is not None)
c = d.conn.cursor()
cc = c.execute('SELECT * FROM dataTable')
cols = list(map(lambda x: x[0], cc.description))
assert(cols == ['module', 'node', 'name', 'label', 'n_index', 'data'])
d.conn.close()
def test_execute_exect_one():
db = StatsDatabase(testing=True)
db.load_data_castalia(castalia_output_file())
with pytest.raises(Exception):
db.execute_expect_one("SELECT node,data FROM dataTable WHERE name = 'Consumed Energy'")
assert db.execute_expect_one("SELECT node,data FROM dataTable WHERE name = 'Consumed Energy' AND node = 0") == (0, 6.79813)
def test_getdatatable():
d = StatsDatabase(testing=True)
d._StatsDatabase__save_output("module1", 1, -1, "outputname", "simlabel", "label", 123.123, table_name="dataTable")
d._StatsDatabase__save_output("module2", 2, -1, "outputname2", "simlabel2", "label2", 1234.1234, table_name="dataTable")
res = d.get_datatable()
assert(res[0] == ("module1", 1, "outputname", "label", -1, 123.123))
assert(res[1] == ("module2", 2, "outputname2", "label2", -1, 1234.1234))
d.conn.close()
def test_createview():
d = StatsDatabase(testing=True)
d.load_data_castalia(castalia_output_file())
d.create_view("test_view", "SELECT node FROM dataTable")
c = d.conn.cursor()
res = c.execute(d.relations["test_view"].sql_fetch_all()).fetchall()
assert res == [(0,), (0,), (0,), (0,), (0,), (0,),
(0,), (0,), (0,), (0,), (0,), (0,),
(0,), (0,), (0,), (0,), (0,), (1,),
(1,), (2,), (2,)]
#TODO check column types in Relation attributes (not yet functional)
d.conn.close()
def test_plot(tmp_dir):
curdir = os.getcwd()
# change dir so that the generated plots will go into that dir
os.chdir(tmp_dir)
d = StatsDatabase(testing=True)
d.load_data_castalia(castalia_output_file())
d.create_view("test_view", "SELECT node, name, data FROM dataTable")
plot = make_plot(d.relations["test_view"], (Column("node"), ),
(Column("data"), ), ["name"],
select={"name": "Consumed Energy"},
terminal=PNG())
plot.make_plot()
# restore the working directory to its previous value
os.chdir(curdir)
def test_add_node_type_parameter():
db = StatsDatabase(testing=True)
db.load_data_castalia(castalia_output_file())
# node parameter
q = Operator(EQ, [
ColumnRef(DATA_TABLE.col["name"]),
ConstantExpr("\"Consumed Energy\"")
])
rel = DerivedTable("test", [Column("node"), Column("data")], [DATA_TABLE],
q)
create_view_for_derived(db, rel)
pm_node = PlotModel("parameter",
"node",
rel, (rel.col["node"], ), (rel.col["data"], ),
select={"name": "Consumed Energy"},
title="Consumed Energy")
# create JsonOutput object
jo = JsonOutput("simulation_results", "243241234143141")
# fetch stats from db
stats = db.execute(
"SELECT node, data FROM dataTable WHERE name = \"Consumed Energy\"")
# add parameters to jsonOutput object
add_node_type_parameter(jo, pm_node, stats)
correct_json = """{
"node_2_node_results": [],
"node_parameter_results": [
{
"results": [
{
"name": "Consumed Energy",
"unit": "",
"value": 6.79813
}
],
"node_id": "0"
},
{
"results": [
{
"name": "Consumed Energy",
"unit": "",
"value": 6.28785
}
],
"node_id": "1"
},
{
"results": [
{
"name": "Consumed Energy",
"unit": "",
"value": 6.28569
}
],
"node_id": "2"
}
],
"simulation_id": "243241234143141",
"node_plot_results": [],
"type": "simulation_results",
"network_plot_results": [],
"network_parameter_results": []
}"""
assert json.loads(correct_json) == json.loads(jo.get_json_string())
def test_create_plot_for_model(tmp_dir):
curdir = os.getcwd()
# change dir so that the generated plots will go into that dir
os.chdir(tmp_dir)
jo = JsonOutput("simulation_results", "SIM1391536771845")
#
# network plot
#
ds = StatsDatabase(testing=True)
ds.load_data_castalia(castalia_output_file())
table_filter = Operator(EQ, [
ColumnRef(DATA_TABLE.col["name"]),
ConstantExpr("\"Consumed Energy\"")
])
dt = DerivedTable(
"test_create_plot_for_model",
[Column("node"), Column("data"),
Column("name")], [DATA_TABLE], table_filter)
create_view_for_derived(ds, dt)
x = DATA_TABLE.col["node"]
y = DATA_TABLE.col["data"]
axes = None
select = {"name": "Consumed Energy"} # all ?
pm1 = PlotModel("plot", "network", dt, (x, ), (y, ), axes, select)
#
# node parameter
#
q = Operator(EQ, [
ColumnRef(DATA_TABLE.col["name"]),
ConstantExpr("\"Consumed Energy\"")
])
rel = DerivedTable("nodepar_test",
[Column("node"), Column("data")], [DATA_TABLE], q)
create_view_for_derived(ds, rel)
pm2 = PlotModel("parameter",
"node",
rel, (rel.col["node"], ), (rel.col["data"], ),
title="Consumed Energy")
#
# network Parameter
#
# column name
c_name = Column("name")
# column SUM(data) AS data
c_data = ColumnExpr("data", Operator(SUM, [ConstantExpr("\"data\"")]))
# filter --> name = "Consumed Energy"
f = Operator(EQ, [
ColumnRef(DATA_TABLE.col["name"]),
ConstantExpr("\"Consumed Energy\"")
])
# create derived table containing the SUM of all "Consumed Energy"
dt = DerivedTable("networkpar_test", [c_name, c_data], [DATA_TABLE], f,
[DATA_TABLE.col["name"]])
# generate a view based on the above derived table
create_view_for_derived(ds, dt)
pm3 = PlotModel("parameter",
"network",
dt,
None, (dt.col["data"], ),
title="Total Consumed Energy")
#
# node2node parameter
#
# DerivedTable for node2node parameter
q = Operator(EQ, [
ColumnRef(DATA_TABLE.col["name"]),
ConstantExpr("\"Packets received per node\"")
])
dt = DerivedTable(
"testn2n",
[Column("node"), Column("n_index"),
Column("data")], [DATA_TABLE], q)
pm4 = PlotModel("parameter",
"node2node",
dt, (dt.col["node"], dt.col["n_index"]),
(dt.col["data"], ),
title="Packets received per node")
# generate a view based on the above derived table
create_view_for_derived(ds, dt)
create_plot_for_model(pm1, ds, jo)
create_plot_for_model(pm2, ds, jo)
create_plot_for_model(pm3, ds, jo)
create_plot_for_model(pm4, ds, jo)
correct_json = """{
"node_parameter_results": [
{
"results": [
{
"name": "Consumed Energy",
"value": 6.79813,
"unit": ""
}
],
"node_id": "0"
},
{
"results": [
{
"name": "Consumed Energy",
"value": 6.28785,
"unit": ""
}
],
"node_id": "1"
},
{
"results": [
{
"name": "Consumed Energy",
"value": 6.28569,
"unit": ""
}
],
"node_id": "2"
}
],
"node_2_node_results": [
{
"parameters": [
{
"unit": "",
"name": "Packets received per node",
"value": 60.0
}
],
"plots": [],
"node1_id": 0,
"node2_id": 1
},
{
"parameters": [
{
"unit": "",
"name": "Packets received per node",
"value": 71.0
}
],
"plots": [],
"node1_id": 0,
"node2_id": 2
}
],
"network_plot_results": [
{
"name": "node over data",
"file_id": "test_create_plot_for_model:data(node).WHERE.name='Consumed.Energy'.png"
}
],
"node_plot_results": [],
"simulation_id": "SIM1391536771845",
"network_parameter_results": [
{
"name": "Total Consumed Energy",
"value": 19.371669999999998,
"unit": ""
}
],
"type": "simulation_results"
}"""
assert json.loads(correct_json) == json.loads(jo.get_json_string())
# restore the working directory to its previous value
os.chdir(curdir)
def test_createviewforderived():
d = StatsDatabase(testing=True)
alist = [
Attribute("a", "int"),
Attribute("b", "int"),
Attribute("c", "int"),
Attribute("d", "int"),
Attribute("e", "int"),
Attribute("f", "int"),
]
ca = Column("a")
cb = Column("b")
cc = Column("c")
cd = Column("d")
ce = Column("e")
cf = Column("f")
t = Table("t", [ca, cb, cc, cd, ce, cf])
d.create_table("t", alist)
dt1_filter = Operator(LAND, [
Operator(NOTEQ, [ColumnRef(ca), ConstantExpr(0)]),
Operator(GREATER, [ColumnRef(ca), ConstantExpr(5)])
])
dt2_filter = Operator(LOR, [
Operator(
LESS,
[Operator(PLUS, [ColumnRef(cc), ConstantExpr(2)]),
ColumnRef(cd)]),
Operator(LESS_EQ, [ColumnRef(cc), ConstantExpr(5)])
])
dt1 = DerivedTable("dt1", [ca, cb], [t], dt1_filter)
dt2 = DerivedTable("dt2", [cc, cd], [t], dt2_filter, [cc])
dt3_filter = Operator(LAND, [
Operator(LOR, [
Operator(LESS, [
Operator(PLUS, [ColumnRef(ca), ConstantExpr(2)]),
ColumnRef(cd)
]),
Operator(LESS_EQ, [ColumnRef(cd), ConstantExpr(5)])
]),
Operator(EQ, [ColumnRef(ca), ColumnRef(cd)])
])
dt3 = DerivedTable("dt3", [dupc(ca), dupc(cd)], [dt1, dt2], dt3_filter,
[ca])
create_view_for_derived(d, dt1)
create_view_for_derived(d, dt2)
create_view_for_derived(d, dt3)
assert d.relations["dt1"].name == "dt1"
dt1_attr = d.get_attributes_of_relation("dt1")
assert dt1_attr[0].name == "a"
assert dt1_attr[1].name == "b"
assert d.relations["dt2"].name == "dt2"
dt2_attr = d.get_attributes_of_relation("dt2")
assert dt2_attr[0].name == "c"
assert dt2_attr[1].name == "d"
assert d.relations["dt3"].name == "dt3"
dt3_attr = d.get_attributes_of_relation("dt3")
assert dt3_attr[0].name == "a"
assert dt3_attr[1].name == "d"
def test_add_node2node_type_parameter():
db = StatsDatabase(testing=True)
db.load_data_castalia(castalia_output_file())
# DerivedTable for node2node parameter
q = Operator(EQ, [
ColumnRef(DATA_TABLE.col["name"]),
ConstantExpr("\"Packets received per node\"")
])
dt = DerivedTable(
"testn2n",
[Column("node"), Column("n_index"),
Column("data")], [DATA_TABLE], q)
# node2node parameter
pm_node = PlotModel("parameter",
"node2node",
dt, (dt.col["node"], dt.col["n_index"]),
(dt.col["data"], ),
title="Packets received per node")
# generate a view based on the above derived table
create_view_for_derived(db, dt)
# create JsonOutput object
jo = JsonOutput("simulation_results", "5678657421")
# fetch stats from db
stats = db.execute("SELECT node,n_index,data FROM testn2n")
# add parameter to jsonOutput object
add_node2node_type_parameter(jo, pm_node, stats)
correct_json = """{
"network_parameter_results": [],
"node_plot_results": [],
"network_plot_results": [],
"simulation_id": "5678657421",
"type": "simulation_results",
"node_parameter_results": [],
"node_2_node_results": [
{
"plots": [],
"node2_id": 1,
"node1_id": 0,
"parameters": [
{
"name": "Packets received per node",
"value": 60.0,
"unit": ""
}
]
},
{
"plots": [],
"node2_id": 2,
"node1_id": 0,
"parameters": [
{
"name": "Packets received per node",
"value": 71.0,
"unit": ""
}
]
}
]
}"""
assert json.loads(correct_json) == json.loads(jo.get_json_string())
def test_plot2json():
#name = "Consumed Energy"
ds = StatsDatabase(testing=True)
ds.load_data_castalia(castalia_output_file())
table_filter = Operator(EQ, [
ColumnRef(DATA_TABLE.columns[3]),
ConstantExpr("\"Consumed Energy\"")
])
dt = DerivedTable(
"test_plot2json1",
[Column("node"), Column("data"),
Column("name")], [DATA_TABLE], table_filter)
create_view_for_derived(ds, dt)
x = DATA_TABLE.col["node"]
y = DATA_TABLE.col["data"]
axes = None
select = {"name": "Consumed Energy"} # all ?
# create a plotmodel based on the above info
pm1 = PlotModel("plot",
"network",
dt, (x, ), (y, ),
axes,
select,
title="node over data")
# dummy plot model for testing
pm2 = PlotModel("plot",
"node",
dt, (x, ), (y, ),
axes,
select,
title="node over data")
# create JsonOutput object
jo = JsonOutput("simulation_results", "07987986875")
plot2json(jo, pm1, "file1")
plot2json(jo, pm2, "file2")
correct_json = '''{
"type": "simulation_results",
"node_parameter_results": [],
"simulation_id": "07987986875",
"node_2_node_results": [],
"network_parameter_results": [],
"network_plot_results": [
{
"name": "node over data",
"file_id": "file1"
}
],
"node_plot_results": [
{
"results": [
{
"name": "node over data",
"file_id": "file2"
}
],
"node_id": "?"
}
]
}'''
assert json.loads(correct_json) == json.loads(jo.get_json_string())
def test_isint():
assert(StatsDatabase._StatsDatabase__is_int(123.0) is True)
assert(StatsDatabase._StatsDatabase__is_int(123.123) is False)
def test_execute():
db = StatsDatabase(testing=True)
db.load_data_castalia(castalia_output_file())
assert db.execute("SELECT node,data FROM dataTable WHERE name = 'Consumed Energy'") == [(0, 6.79813), (1, 6.28785), (2, 6.28569)]
def test_getnodes():
d = StatsDatabase(testing=True)
d.load_data_castalia(castalia_output_file())
assert d.get_nodes() == [0, 1, 2]