def make_plot(self, gnuplot="gnuplot"):
p = Popen(gnuplot, stdin=PIPE, stderr=PIPE)
ret = False
try:
script = self.__create_script()
if script:
p.stdin.write(script.encode("utf-8"))
ret = True
# vsam: changed timeout to 10 sec (it was 0.2 but I was getting timeouts)
out, err = p.communicate(timeout=10)
err_str = err.decode("utf-8").splitlines()[:10] if err else ""
err_str = "\n".join(err_str)
if err_str != "" and re.search("gnuplot>\ ", err_str):
if os.path.isfile(self.output + ".png"):
os.remove(self.output + ".png")
fail("generation failed, gnuplot reported error:\n%s" %
err_str)
except CheckFail as ex:
raise ex
except BaseException as ex:
logging.critical(traceback.format_exc())
fail("generation failed")
finally:
p.stdin.close()
return ret
def gen_plotmodel(rel, d):
"""
generate a PlotModel associated to relation rel from specified dictionary d
(the dictionary should represent only one PlotModel)
returns the PlotModel
"""
sel = ViewsPlotsDecoder.get_attr("select", d)
if sel != pm_defaults["select"]:
sel = SelectorParser.parse(sel, rel)
if "model_type" not in d:
fail("\"model_type\" must be specified")
if "stat_type" not in d:
fail("\"stat_type\" must be specified")
pm = PlotModel(d["model_type"], d["stat_type"], rel,
ViewsPlotsDecoder.get_attr("x", d, rel),
ViewsPlotsDecoder.get_attr("y", d, rel),
ViewsPlotsDecoder.get_attr("axes", d), sel,
ViewsPlotsDecoder.get_attr("title", d),
ViewsPlotsDecoder.get_attr("style", d),
ViewsPlotsDecoder.get_attr("legend", d),
ViewsPlotsDecoder.get_attr("xlabel", d),
ViewsPlotsDecoder.get_attr("ylabel", d),
ViewsPlotsDecoder.get_attr("x_range", d),
ViewsPlotsDecoder.get_attr("y_range", d),
ViewsPlotsDecoder.get_attr("logscale", d),
ViewsPlotsDecoder.get_attr("grid", d),
ViewsPlotsDecoder.get_attr("key", d),
ViewsPlotsDecoder.get_attr("unit", d))
return pm
def get_func_by_name(name):
funcs = [AVG, COUNT, MAX, MIN, SUM, LAND, LOR]
for f in funcs:
if name.lower() == f.name.lower():
return f
if name == "Eq":
return EQ
elif name == "NotEq":
return NOTEQ
elif name == "Lt":
return LESS
elif name == "LtE":
return LESS_EQ
elif name == "Gt":
return GREATER
elif name == "GtE":
return GREATER_EQ
elif name == "Add":
return PLUS
elif name == "Sub":
return MINUS
elif name == "Mult":
return MULT
elif name == "Div":
return DIV
fail("unknown function: \"%s\"" % name)
def normalize_index(self,shape, idx):
'''
Remove '...' and replace with sequence of ':'
'''
# (a) normalize: remove any ...
n = idx.count(...) # number of ellipses
d = idx.count(None) # number of new dims
if n>1:
fail("error: index contains more than one ellipse (...)")
if n==1:
# replace ellipse
p = idx.index(...)
else:
# expand at the end if needed
assert n==0
p = len(idx)
before = idx[0:p]
after = idx[p+1:]
l = len(shape)+d-len(before)-len(after)
if l<0:
fail("error: too many coordinates in index")
idx = before+[slice(None,None,None)]*l+after
assert len(idx)==len(shape)+d
return idx
def transform_id(ast, scope):
qual_id = ast[1:]
# Named constants
if len(qual_id) == 1:
if qual_id[0] == 'nan': return Literal(float('nan'))
if qual_id[0] == 'inf': return Literal(float('inf'))
# Look up in scope
try:
obj = scope[qual_id]
except KeyError:
fail("error: name %s does not exist in scope" % ('.'.join(qual_id)))
# Found object, now make it into an expression,
# if applicable.
if isinstance(obj, Variable):
return VarRef(obj)
elif isinstance(obj, FExpr):
return obj.expr
elif isinstance(obj, Parameter):
return obj
fail("error: name '%s' is not an expression, it is a %s",
'.'.join(qual_id), obj if isinstance(obj, SourceItem) else 'builtin')
def model2plots(pml, jo, castalia_data=None):
"""Accepts a list of PlotModel objects and creates the corresponding plots/parameters(statistics).
"""
assert isinstance(pml, list)
assert all(isinstance(pm, PlotModel) for pm in pml)
# Collect a list of tables, ordered according to dependence
table_list = collect_tables_for_pml(pml)
# Create database
ds = StatsDatabase()
# create tables
for table in table_list:
if isinstance(table, Table) and not isinstance(table, DerivedTable):
with Context(view=table.name):
create_table(ds, table)
populate_table(ds, table, castalia_data)
# create views
for table in table_list:
if isinstance(table, DerivedTable):
with Context(view=table.name):
create_view_for_derived(ds, table)
# create plots
for pm in pml:
if pm.rel.name in ds.relations:
with Context(view=pm.rel.name, plot=pm.title):
create_plot_for_model(pm, ds, jo)
else:
fail(
"View %s does not exist in database, some error occurred during its generation"
% pm.rel.name)
def __init__(self, model, evt, stmt):
try:
self.model = model
self.event = evt
self.statement = stmt
except:
fail()
def __init__(self, scope, name, rtype, expr, constdecl):
super().__init__(scope, name)
self.rtype = rtype
self.expr = expr
if constdecl > expr.const:
fail("error: expected a constant expression for '%s'", name)
self.constdecl = constdecl
def visit_Attribute(self, node):
p = self.visit(node.value)
col_name = node.attr
if col_name not in self.types:
fail("Unknown Column name: \"%s\"" % col_name)
c = Column(col_name, Table(p, []))
c.table = Table(p, [])
return ColumnRef(c)
def __init__(self, *args):
self.args = list(args)
for arg in self.args:
if not isinstance(arg, (str, ExprNode)):
fail(
"Illegal argument list (only strings and expressions are allowed",
ooc=ValueError)
def validate_instance(self, nodeType):
"Check datarate and phyFrameOverhead"
self_p = self.phyFrameOverhead
radio_p = nodeType.comm.Radio.phyFrameOverhead
if self_p != radio_p:
fail(
"MAC(CC2420Mac) phyFrameOverhead(=%d)"
" is not the same as that of the radio(=%d)", self_p, radio_p)
def get_table_by_name(self, name):
"""
returns the Table with specified name
"""
assert isinstance(name, str)
for c in self.derived_tables:
if c.name == name:
return c
fail("View \"%s\" does not exist" % name)
def assert_format():
"""
checks table column number to be equal with column number in data file
"""
file_cols = len(row)
if file_cols != table_cols:
fail(
"data file (\"%s\") format (%d columns) does not match table (\"%s\") format (%d columns)"
% (filename, file_cols, table.name, table_cols),
ooc=TypeError)
def process_normal(self, S, s):
self.check_scalar_int(S)
if S.const:
pos = S.value
if pos>s or pos<-s-1:
fail("Index out of range")
self.impl_expr.append("%d" % pos)
else:
v = self.getvar(S)
self.impl_expr.append(v)
def create_view_for_derived(ds, dt):
"""
Create an SQL view in ds for given DerivedTable dt.
"""
sql = derived2sql(dt)
try:
ds.create_view(dt.name, sql)
except BaseException as ex:
# this should be here to catch unhandled sql syntax errors
fail(ex)
def result_dtype(self):
s = self.std_result_dtype()
s = s.char * self.arity
tt = ''.join(s) + "->"
for m in self.ufunc.types:
if m.startswith(tt):
rt=m[len(tt):]
return np.dtype(rt)
fail("error: illegal operand types")
def visit_Name(self, node):
name = str(node.id)
if name in self.types:
if self.types[name] == "function":
return name
elif self.types[name] == "column":
return ColumnRef(Column(name))
elif self.types[name] == "table":
return name
else:
fail("Unknown Name: \"%s\"" % name)
def bind(self, name, obj, force=False):
'''
Bind the name to obj in this scope. If force is False and the name
is already bound, a KeyError is raised.
'''
if obj is None:
raise ValueError("Illegal attempt to bind None to a name")
if not is_legal_identifier(name):
raise KeyError("Name {0} is not a legal identifier".format(name))
if not force and name in self.symtab:
fail("Name '%s' bound in scope." % name, ooc=KeyError)
self.symtab[name] = obj
def populate_table(ds, table, castalia_data=None):
"""
load data appropriate for this table in database ds
if castalia_data is set, override the table's filename
"""
if table.format == "dataTable":
ds.load_data_castalia(
table.filename if castalia_data is None else castalia_data,
table.name)
elif table.format == "csv":
ds.load_data_csv(table)
else:
fail("unknown format %s" % table.format)
def get_select_columns(col):
if isinstance(col, ColumnExpr):
ret = expression2sql(col, prec_table=True)
if ret is None:
# This should only happen if there was some error in expression parsing, and the expression passed to
# expression2sql is malformed
fail("error in views generation")
else:
if col.origin_table:
ret = col.origin_table.name + "." + col.name
else:
ret = col.name
return ret
def str_2_expr(expr_str, types):
"""
Parses the expression given in expr_str to an actual Expression object
returns the generated Expression
"""
nv = ExprGenNodeVisitor(types)
eq_regex = re.compile(
r"(?<![=><])=(?![=><])") # regular expression to find a single =
expr_str = eq_regex.sub("==", expr_str) # replaces = with ==
try:
node = ast.parse(expr_str)
return nv.visit(node)
except SyntaxError as ex:
fail("Syntax Error: %s" % ex.text)
def transform_params(plist, scope, shape=None):
params = []
for p in plist:
assert isinstance(p, AstNode)
_, typename, name = p
param = Parameter(name=name,
type=TypeInfo.forName(typename),
shape=shape)
try:
scope.bind(name, param)
except KeyError:
fail("error: multiple uses of name '%s' in parameter list", name)
param.ast = p
params.append(param)
return params
def col_str2col_obj(col_str, col_obj):
"""
returns a list of Column objects, defined by the list of column names
col_str: the list of column names that we want to transform in Column objects
col_obj: the list of all Column objects for one DerivedTable
"""
assert isinstance(col_str, list)
assert isinstance(col_obj, list)
cols = []
for s in col_str:
c = get_col_by_name(s, col_obj)
if c:
cols.append(c)
else: # this should never happen
fail("column \"%s\" does not exist" % s)
return cols
def get_attr(attr, d, rel=None):
"""
helper function to return the needed argument from dictionary d if it exists or get a default value for it
rel is needed only for "x" and "y". It is the Table that columns in x and y belong
"""
if attr in d and d[attr] != "" and d[attr] != []:
if attr in ["x", "y"]:
xy = d[attr]
if xy is None: return None
return ViewsPlotsDecoder.xy_to_valid_col_tuple(xy, rel)
else:
return d[attr]
elif attr in pm_defaults:
return pm_defaults[attr]
else:
fail("Bad argument \"%s\"" % attr)
def parse(selector_text, rel, testing=None):
assert isinstance(selector_text, str)
assert isinstance(rel, Table)
colnames = {col.name for col in rel.columns}
namespace = dict(SelectorParser.allowed_funcs)
try:
SelectorParser.validate(selector_text, colnames)
namespace.update({name: name for name in colnames})
selector = eval("{" + selector_text + "}", {},
SelectorParser.StrictDict(namespace))
except:
fail("The selector '%s' is malformed" % selector_text,
ooc=ValueError)
return selector
def __init__(self, scope, name, type, initval, toplevel=True):
super().__init__(scope, name)
try:
self.type = type
self.initval = initval
self.toplevel = toplevel
except:
fail()
(initval.is_proper() and initval.const) or \
fail("error: variable '%s' initialized by non-constant", name)
initval.auto_castable(type) or \
fail("error: cannot assign to '%s' from incopatible type '%s'",
type.name, initval.type.name)
# Find the model you belong to
self.model = scope
def __getitem__(self, name):
# reimplement to get qualified names
if isinstance(name, tuple):
scope = self
for n in name:
assert isinstance(scope, Scope)
scope = scope[n]
return scope
else:
seq = name.split('.')
if len(seq) == 1:
obj = self.lookup(name)
if obj is None:
fail("error: name '{0}' undefined in scope".format(name),
ooc=KeyError)
return obj
else:
return self[tuple(seq)]
def derived2sql(dt):
"""
generates an sql query to be used in the creation of the derived table as a view
SELECT ... FROM ... WHERE ... GROUPBY
"""
assert isinstance(dt, DerivedTable)
sql = "SELECT " + ",".join(list(map(get_select_columns, dt.columns)))
sql += " FROM " + ",".join(list(map(lambda x: x.name, dt.base_tables)))
if dt.table_filter:
where_expr = expression2sql(dt.table_filter)
if where_expr is None:
# This should only happen if there was some error in expression parsing, and the expression passed to
# expression2sql is malformed
fail("error in views generation")
sql += " WHERE " + where_expr
if dt.groupby:
sql += " GROUP BY " + ",".join(list(map(lambda x: x.name, dt.groupby)))
return sql
def result_shape(self):
# argument shapes must broadcast on all coordinates except the
# first.
if not self.args_proper_shape(): return None
shapes = set()
D = 0
for arg in self.args:
shape = arg.shape
if shape is None:
return None
if shape == tuple():
fail("concatenation of scalars is not supported")
D += shape[0]
shapes.add(shape[1:])
if len(shapes)==1:
# all operands have same residual shape!
self.all_similar = True
return (D,)+shapes.pop()
return (D,)+compute_broadcast_shape(shapes)
def instantiate(self, *args):
'''Instantiate a function call for the
given arguments'''
# check no. of arguments
if len(args) != len(self.params):
fail("error: in call to '%s', expected %d arguments, got %d",
self.name, len(self.params), len(args))
argmap = {}
for param, arg in zip(self.params, args):
if not arg.type.auto_castable(param.type):
fail(
"error: in call to '%s', wrong type for parameter %s, expected %s, got %s",
self.name, param.name, param.type.name, arg.type.name)
argmap[param.name] = arg
return self.expression.bind(argmap)
