def generate_simulation(fileloc=None, loglevel=logging.DEBUG, raises=True, validate=False):
"""
This function is called by the executor to bootstrap code generation.
"""
from models.validation import inform, warn
with GenProcess(loglevel) as genproc:
# Get the root object
try:
context.validate = validate
dstore = context.datastore
sim = context.datastore.get_root_object()
except Exception as e:
logger.exception("In creating datastore", exc_info=1)
inform("Created root object", sim)
# From the root object, select the proper Generator class
GenCls = GENERATOR_REGISTRY[sim['generator']]
# run the generator
generator = GenCls()
generator.generate()
result = {
'success': genproc.success,
'messages': genproc.messages,
'nsd_id': sim['nsdid']
}
if not genproc.success and raises:
raise GenError(result)
else:
return result
def validate_output():
"""
Validate the NSD and create the plot models
"""
context.validate = True
transform_nsd_plots()
inform("NSD data analysis validated.")
def gen_plots(self, rel, d_plot_list):
"""
generate PlotModels from d_plot_list which is a list of dictionaries representing the plots
then add the generated PlotModels to plot_models (a list of PlotModels)
rel is the relation these plots are connected with
"""
for p in d_plot_list:
with Context(plot=p["title"]):
pm = self.gen_plotmodel(rel, p)
self.plot_models.append(pm)
inform("plot validated.")
def create_plot_for_model(pm, ds, jo):
"""
Create a plot/parameter(statistic) for given PlotModel and StatsDatabase ds
add generated values to JsonOutput jo
"""
assert isinstance(pm, PlotModel)
terminal = PNG()
output = "DEFAULT"
plot = make_plot(ds.relations[pm.rel.name], pm.x, pm.y, pm.axes, pm.select,
pm.title, pm.style, pm.legend, pm.xlabel, pm.ylabel,
pm.x_range, pm.y_range, terminal, pm.logscale, pm.grid,
pm.key, output)
if pm.title == "DEFAULT":
if pm.model_type == "plot":
pm.title = default_title(
pm.x, pm.y, pm.axes if pm.axes else [],
{axis: set([])
for axis in pm.axes} if pm.axes else None).lstrip().rstrip()
elif pm.model_type == "parameter":
# no default title for parameters
pass
if pm.model_type == "plot":
# generate the plot to the current working directory
if plot.make_plot():
# add plot to JsonOutput jo
plot2json(jo, pm, plot.output + ".png")
inform("generated successfully")
elif pm.model_type == "parameter":
# generate the parameter (statistic)
res = plot.make_parameter()
if len(res) != 0 and res[0] != (None, ):
# add the parameter to JsonOutput jo
parameter2json(jo, pm, res)
inform("generated sucessfully")
else:
warn("no data found")
else:
logging.error("invalid model type: \"%s\"" % pm.model_type)
fail("invalid model type: \"%s\"" % pm.model_type)
def decode(self, views):
"""
Decodes views and plots in json format to DerivedTable and PlotModel respectively
returns a tuple of lists (list_DerivedTable, list_plotModel)
"""
for v in views:
if "name" not in v:
fail("Malformed View, is missing name")
if "columns" not in v:
fail("Malformed View \"%s\", is missing columns" % v["name"])
with Context(view=v["name"]):
allowed_chars = re.compile(r"^[a-zA-Z0-9_]+$")
if not allowed_chars.match(v["name"]):
fail(
"View name can contain only upper/lower case letters, numbers and underscores"
)
# For backward compatibility
if v["name"] == "dataTable":
v.setdefault("filename", "simout.txt")
v.setdefault("format", "dataTable")
v.setdefault("node_mapping", ["node", "n_index"])
for c in v["columns"]:
if "type" not in c:
if c["name"] == "data":
c["type"] = "real"
else:
c["type"] = "varchar"
if "filename" in v and v["filename"] != "":
rel = self.gen_table(v)
else:
rel = self.gen_derived_table(v)
if "plots" in v:
self.gen_plots(rel, v["plots"])
inform("View validated.")
return self.derived_tables, self.plot_models
def get_model(self, name):
'''
Return a model by the given name from this factory.
'''
model = self.lookup(name)
if model is None:
with self.process_factory(name=name) as c:
# May throw
src = self.get_model_source(name)
# insert a dummy entry into the symtab, (used to check for cycles)
self.bind(name, 'forward')
model = self.__compile(name, src)
if not c.success or model is None:
warn("Compilation failed for model %s", name)
else:
inform("Model %s is compiled successfully", model.name)
self.add_model(model, force=True)
elif model == 'forward':
fatal("cyclical reference for module %s", name)
return model
def transform_nsd_plots():
filename = "nsd.json"
vpd = ViewsPlotsDecoder()
with open(filename, "r") as f:
json_str = f.read()
nsd = json.loads(json_str)
if "views" not in nsd:
inform("There are no output definitions in the NSD")
nsd["views"] = []
#
# DerivedTable, PlotModel creation from nsd
#
# if this fails we abort result generation
# this is what happens when we validate the nsd
#
derived_tables, plot_models = vpd.decode(nsd["views"])
return plot_models
def process_vectorl(project_id, model_name, run=False, until=None, steps=None):
'''
Compile and return results.
'''
output_list = []
pf = ProcProcess.new_factory(output_list)
factory = ProjectModelFactory(project_id, pf)
varlist = []
with pf(name='top process', logger=None) as top:
top.suppress(Exception)
factory.get_model(model_name)
if top.success:
varlist = [
v.full_name for v in factory.all_variables() if v.toplevel
]
inform("Got model")
logging.info("Output list: %s", output_list)
comp_output = {
'type': 'vectorl_compiler_output',
'project_id': project_id,
'vectorl_model_name': model_name,
'id_map': factory.id_map,
'vectorl_object_map': factory.object_map,
'success': top.success,
'compiler_output': list(output_list), # make a copy!
"variables": varlist
}
if not run:
return comp_output
# we are also asked to run the model
result = {'compiler': comp_output}
if not top.success:
return result
# ok, we compiled correctly, now we need to run!
# first, we empty the output list
del output_list[:]
assert top.success
runner = None
with top:
runner = Runner(factory)
runner.start(until, steps)
result.update({
'type': 'vectorl_run_output',
'success': top.success,
'run_output': output_list
})
if runner is not None:
result.update({
'maximum_steps': str(steps),
'maximum_time': str(until),
'run_steps': runner.step,
'end_time': runner.now
})
return result