def test_lag_variable(self): from opus_core.variables.variable_factory import VariableFactory from opus_core.variables.variable_name import VariableName vf = VariableFactory() var_name = VariableName('opus_core.tests.a_test_variable_lag3') var = vf.get_variable(var_name, None, index_name='my_id') self.assert_(var.is_lag_variable()) self.assertEqual(var.lag_offset, 3)
class DependencyQuery: def __init__(self, config, model=None, model_group=None, specification=None, scenario_name=None): self.factory = VariableFactory() lib = config.get_expression_library() self.factory.set_expression_library(lib) self.model = model self.model_group = model_group if model is not None: if specification is None: specification_dict = config.get_estimation_specification( model, model_group) spec = get_specification_for_estimation(specification_dict) else: spec = specification model_prefix = '' if model_group is not None: model_prefix = '%s_' % model_group self.model_name = '%s%s' % (model_prefix, model) self.var_list = spec.get_distinct_long_variable_names().tolist() #check other model nodes, such as agents_filter, submodel_string or filter config_node_path = "model_manager/models/model[@name='%s']" % self.model model_node = config._find_node(config_node_path) controller = config._convert_model_to_dict(model_node) addvars = [] addvars.append( controller.get('init', {}).get('arguments', {}).get('filter', None)) addvars.append( controller.get('init', {}).get('arguments', {}).get('submodel_string', None)) addvars.append( controller.get('init', {}).get('arguments', {}).get('choice_attribute_name', None)) addvars.append( controller.get('prepare_for_run', {}).get('arguments', {}).get('agent_filter', None)) for var in addvars: if isinstance(var, str): self.var_list.append( eval(var) ) # eval because these entries are in double quotes, e.g. "'attribute'" self.var_tree = [] l = [] for var in self.var_list: l.append((var, [])) self.var_tree.append((self.model_name, l)) else: # this is meant to be for all models but is not working yet #self.config = config.get_run_configuration(scenario_name) self.var_list = [] self.var_tree = [] #TODO: there seems to be an issue with the (xml)dictionary approach -there can be # multiple, indexed, submodels. This only seems to retrieve the first #this collects all the variables models depend on #self.var_tree = [] # self.var_list = [] # #TODO: if we update to ElementTree 1.3, use # # model_manager/model_system//specification/[@type='submodel']/variables # for x in config._find_node('model_manager/models//specification'): # l = [] # for y in x: # if y.get('type') == 'submodel': # t = y.find('variable_list') # if len(t) > 0: # for z in config._convert_variable_list_to_data(t[0]): # for k in lib.keys(): # if k[1] == z: # l.append((lib[k], [])) # self.var_list.append(lib[k]) # self.var_tree.append((x, l)) #given a name, return an instance of Variable def get_var(self, name): #creates a fake dataset, required for variable resolution def create_fake_dataset(dataset_name): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='fake_dataset', table_data={'id': array([], dtype='int32')}) dataset = Dataset(in_storage=storage, in_table_name='fake_dataset', dataset_name=dataset_name, id_name="id") return dataset var = VariableName(name) dataset = var.get_dataset_name() try: return self.factory.get_variable(var, create_fake_dataset(dataset), quiet=True) except LookupError: #print "LOOKUP ERROR: " + name return None #given a name, returns the tree with the model at the root, and the vars it depends on as leaves def get_model_vars(self, name=None, group=None): if name is None: name = self.model if group is None: group = self.model_group model_prefix = '' if group is not None: model_prefix = '%s_' % group def find(f, seq): for item in seq: if f(item): return item model = find(lambda x: x[0] == '%s%s' % (model_prefix, name), self.var_tree) if model == None: raise "Model " + name + " not found." else: return model # returns a list of VariableNames a model depends on def get_model_var_list(self, name, group=None): ret = [] def rec(xs): for x in xs: ret.append(VariableName(x[0])) rec(x[1]) rec( map(self.get_dep_tree_from_name, extract_leaves(self.get_model_vars(name, group)[1]))) return elim_dups(ret) #get a dependency tree for a variable given its name def get_dep_tree_from_name(self, name): varclass = self.get_var(name) if (varclass == None): return (name, [], "primary") return self.get_dep_tree(varclass) #returns a dependency tree given a particular variable def get_dep_tree(self, inp): result_others = [] dependencies_list = inp.get_current_dependencies() for x in dependencies_list: dep_item = x[0] if isinstance(dep_item, str): result_others.append(self.get_dep_tree_from_name(dep_item)) else: print "Attribute!" return (inp.name(), elim_dups(result_others)) def all_models_tree(self): return map(self.get_dep_tree_from_name, extract_leaves(self.var_tree)) \ + map(lambda x,y: (x, y,"model"), self.var_tree.iteritems()) def model_tree(self): model = self.get_model_vars() return map(self.get_dep_tree_from_name, extract_leaves(model[1])) \ + [(model[0], model[1],"model")] def vars_tree(self, vl): return map(self.get_dep_tree_from_name, extract_leaves(vl))
class DependencyQuery: def __init__(self, config, model=None, model_group=None, specification=None, scenario_name=None): self.factory = VariableFactory() lib = config.get_expression_library() self.factory.set_expression_library(lib) self.model = model self.model_group = model_group if model is not None: if specification is None: specification_dict = config.get_estimation_specification(model, model_group) if model_group is not None: specification_dict = specification_dict[model_group] spec = get_specification_for_estimation(specification_dict) else: spec = specification model_prefix = '' if model_group is not None: model_prefix = '%s_' % model_group self.model_name = '%s%s' % (model_prefix, model) self.var_list = spec.get_distinct_long_variable_names().tolist() #check other model nodes, such as agents_filter, submodel_string or filter # config_node_path = "model_manager/models/model[@name='%s']" % self.model # model_node = config._find_node(config_node_path) # controller = config._convert_model_to_dict(model_node) # addvars = [] # addvars.append(controller.get('init', {}).get('arguments', {}).get('filter', None)) # addvars.append(controller.get('init', {}).get('arguments', {}).get('submodel_string', None)) # addvars.append(controller.get('init', {}).get('arguments', {}).get('choice_attribute_name', None)) # addvars.append(controller.get('prepare_for_run', {}).get('arguments', {}).get('agent_filter', None)) # This assumes that xml nodes contain the tag 'model_dependency_type' self.model_structure_dependencies = config.model_dependencies(self.model) self.var_list = self.var_list + self.model_structure_dependencies.get('variable', []) # for var in addvars: # if isinstance(var, str): # self.var_list.append(eval(var)) # eval because these entries are in double quotes, e.g. "'attribute'" self.var_tree = [] l = [] for var in self.var_list: l.append((var, [])) self.var_tree.append((self.model_name, l)) else: # this is meant to be for all models but is not working yet #self.config = config.get_run_configuration(scenario_name) self.var_list = [] self.var_tree = [] #TODO: there seems to be an issue with the (xml)dictionary approach -there can be # multiple, indexed, submodels. This only seems to retrieve the first #this collects all the variables models depend on #self.var_tree = [] # self.var_list = [] # #TODO: if we update to ElementTree 1.3, use # # model_manager/model_system//specification/[@type='submodel']/variables # for x in config._find_node('model_manager/models//specification'): # l = [] # for y in x: # if y.get('type') == 'submodel': # t = y.find('variable_list') # if len(t) > 0: # for z in config._convert_variable_list_to_data(t[0]): # for k in lib.keys(): # if k[1] == z: # l.append((lib[k], [])) # self.var_list.append(lib[k]) # self.var_tree.append((x, l)) #given a name, return an instance of Variable def get_var(self, name): #creates a fake dataset, required for variable resolution def create_fake_dataset(dataset_name): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='fake_dataset', table_data={ 'id':array([], dtype='int32') } ) dataset = Dataset(in_storage=storage, in_table_name='fake_dataset', dataset_name=dataset_name, id_name="id") return dataset var = VariableName(name) dataset = var.get_dataset_name() try: return self.factory.get_variable(var, create_fake_dataset(dataset), quiet=True) except LookupError: #print "LOOKUP ERROR: " + name return None #given a name, returns the tree with the model at the root, and the vars it depends on as leaves def get_model_vars(self, name=None, group=None): if name is None: name = self.model if group is None: group = self.model_group model_prefix = '' if group is not None: model_prefix = '%s_' % group def find(f, seq): for item in seq: if f(item): return item model = find(lambda x: x[0] == '%s%s' % (model_prefix, name), self.var_tree) if model == None: raise "Model " + name + " not found." else: return model # returns a list of VariableNames a model depends on def get_model_var_list(self, name, group=None): ret = [] def rec(xs): for x in xs: ret.append(VariableName(x[0])) rec(x[1]) rec(map(self.get_dep_tree_from_name, extract_leaves(self.get_model_vars(name, group)[1]))) return elim_dups(ret) #get a dependency tree for a variable given its name def get_dep_tree_from_name(self, name): varclass = self.get_var(name) if(varclass == None): return (name, [], "primary") return self.get_dep_tree(varclass) #returns a dependency tree given a particular variable def get_dep_tree(self, inp): result_others = [] dependencies_list = inp.get_current_dependencies() for x in dependencies_list: dep_item = x[0] if isinstance(dep_item, str): result_others.append(self.get_dep_tree_from_name(dep_item)) else: print "Attribute!" return (inp.name(), elim_dups(result_others)) def all_models_tree(self): return map(self.get_dep_tree_from_name, extract_leaves(self.var_tree)) \ + map(lambda x,y: (x, y,"model"), self.var_tree.iteritems()) def model_tree(self): model = self.get_model_vars() return map(self.get_dep_tree_from_name, extract_leaves(model[1])) \ + [(model[0], model[1],"model")] def vars_tree(self, vl): return map(self.get_dep_tree_from_name, extract_leaves(vl)) def get_model_structure_dependencies(self): return self.model_structure_dependencies