Example #1
0
    def setUp(self):
        hh_link = links.Many2One('household', 'hh_id', 'household')
        mother_link = links.Many2One('mother', 'mother_id', 'person')
        child_link = links.One2Many('children', 'mother_id', 'person')
        persons_link = links.One2Many('persons', 'hh_id', 'person')

        dt = np.dtype([('period', int), ('id', int), ('age', int),
                       ('dead', bool),  ('mother_id', int), ('hh_id', int)])
#TODO: I can't use an EntityContext with an array containing several periods
#      of data
#        persons = array([(2000, 0, 53, False, -1, 0),
#                         (2000, 1, 23, False,  0, 1),
#                         (2000, 2, 20, False,  0, 2),
#                         (2000, 3, 43, False, -1, 3), 
#                         (2001, 0, 54,  True, -1, 0),
#                         (2001, 1, 24, False,  0, 1),
#                         (2001, 2, 21, False,  0, 2),
#                         (2001, 3, 44, False, -1, 0), # they got married 
#                         (2001, 4,  0, False,  2, 2),
        persons = array([(2002, 0, 55,  True, -1, 0),
                         (2002, 1, 25, False,  0, 1),
                         (2002, 2, 22, False,  0, 2),
                         (2002, 3, 45, False, -1, 0),
                         (2002, 4,  1, False,  2, 2)],
                        dtype=dt)
        person = Entity('person',
                        links={'household': hh_link,
                               'mother': mother_link,
                               'children': child_link},
                        array=persons)
#         person = FakeEntity('person',
#                             links={'household': hh_link,
#                                    'mother': mother_link,
#                                    'children': child_link},
#                             data=persons)

        dt = np.dtype([('period', int), ('id', int)])
#        households = array([(2000, 0),
#                            (2000, 1),
#                            (2000, 2),
#                            (2000, 3),
#                             
#                            (2001, 0),
#                            (2001, 1),
#                            (2001, 2),
                            
        households = array([(2002, 0),
                            (2002, 1),
                            (2002, 2)],
                           dtype=dt)
#         household = FakeEntity('household',
#                                links={'persons': persons_link},
#                                data=households)
        household = Entity('household',
                           links={'persons': persons_link},
                           array=households)
        entity_registry.add(person)
        entity_registry.add(household)
Example #2
0
    def from_yaml(cls, fpath,
                  input_dir=None, input_file=None,
                  output_dir=None, output_file=None):
        simulation_path = os.path.abspath(fpath)
        simulation_dir = os.path.dirname(simulation_path)
        with open(fpath) as f:
            content = yaml.load(f)

        content = handle_imports(content, simulation_dir)
        validate_dict(content, cls.yaml_layout)

        # the goal is to get something like:
        # globals_def = {'periodic': [('a': int), ...],
        #                'MIG': int}
        globals_def = {}
        for k, v in content.get('globals', {}).iteritems():
            # periodic is a special case
            if k == 'periodic':
                type_ = fields_yaml_to_type(v)
            else:
                # "fields" and "type" are synonyms
                type_def = v.get('fields') or v.get('type')
                if isinstance(type_def, basestring):
                    type_ = field_str_to_type(type_def, "array '%s'" % k)
                else:
                    if not isinstance(type_def, list):
                        raise SyntaxError("invalid structure for globals")
                    type_ = fields_yaml_to_type(type_def)
            globals_def[k] = type_

        simulation_def = content['simulation']
        seed = simulation_def.get('random_seed')
        if seed is not None:
            seed = int(seed)
            print("using fixed random seed: %d" % seed)
            random.seed(seed)
            np.random.seed(seed)

        periods = simulation_def['periods']
        start_period = simulation_def['start_period']
        config.skip_shows = simulation_def.get('skip_shows', False)
        #TODO: check that the value is one of "raise", "skip", "warn"
        config.assertions = simulation_def.get('assertions', 'raise')
        config.show_timings = simulation_def.get('timings', True)

        autodump = simulation_def.get('autodump', None)
        if autodump is True:
            autodump = 'autodump.h5'
        if isinstance(autodump, basestring):
            # by default autodump will dump all rows
            autodump = (autodump, None)
        config.autodump = autodump

        autodiff = simulation_def.get('autodiff', None)
        if autodiff is True:
            autodiff = 'autodump.h5'
        if isinstance(autodiff, basestring):
            # by default autodiff will compare all rows
            autodiff = (autodiff, None)
        config.autodiff = autodiff

        input_def = simulation_def['input']
        input_directory = input_dir if input_dir is not None \
                                    else input_def.get('path', '')
        if not os.path.isabs(input_directory):
            input_directory = os.path.join(simulation_dir, input_directory)
        config.input_directory = input_directory

        output_def = simulation_def['output']
        output_directory = output_dir if output_dir is not None \
                                      else output_def.get('path', '')
        if not os.path.isabs(output_directory):
            output_directory = os.path.join(simulation_dir, output_directory)
        if not os.path.exists(output_directory):
            print("creating directory: '%s'" % output_directory)
            os.makedirs(output_directory)
        config.output_directory = output_directory

        if output_file is None:
            output_file = output_def['file']
        output_path = os.path.join(output_directory, output_file)

        for k, v in content['entities'].iteritems():
            entity_registry.add(Entity.from_yaml(k, v))

        for entity in entity_registry.itervalues():
            entity.check_links()
            entity.parse_processes(globals_def)
            entity.compute_lagged_fields()

        init_def = [d.items()[0] for d in simulation_def.get('init', {})]
        init_processes, init_entities = [], set()
        for ent_name, proc_names in init_def:
            if ent_name not in entity_registry:
                raise Exception("Entity '%s' not found" % ent_name)

            entity = entity_registry[ent_name]
            init_entities.add(entity)
            init_processes.extend([(entity.processes[proc_name], 1)
                                   for proc_name in proc_names])

        processes_def = [d.items()[0] for d in simulation_def['processes']]
        processes, entity_set = [], set()
        for ent_name, proc_defs in processes_def:
            entity = entity_registry[ent_name]
            entity_set.add(entity)
            for proc_def in proc_defs:
                # proc_def is simply a process name
                if isinstance(proc_def, basestring):
                    # use the default periodicity of 1
                    proc_name, periodicity = proc_def, 1
                else:
                    proc_name, periodicity = proc_def
                processes.append((entity.processes[proc_name], periodicity))
        entities = sorted(entity_set, key=lambda e: e.name)

        method = input_def.get('method', 'h5')

        if method == 'h5':
            if input_file is None:
                input_file = input_def['file']
            input_path = os.path.join(input_directory, input_file)
            data_source = H5Data(input_path, output_path)
        elif method == 'void':
            data_source = Void(output_path)
        else:
            raise ValueError("'%s' is an invalid value for 'method'. It should "
                             "be either 'h5' or 'void'")

        default_entity = simulation_def.get('default_entity')
        return Simulation(globals_def, periods, start_period,
                          init_processes, init_entities, processes, entities,
                          data_source, default_entity)
Example #3
0
    def from_yaml(cls, main_fpath,
                  input_dir=None, input_file=None,
                  output_dir=None, output_file=None):
        simulation_path = os.path.abspath(main_fpath)
        simulation_dir = os.path.dirname(simulation_path)
        
        #TODO : add an instruction in yaml_layout in so as to call an other YAML file and use it instead of a list. 
        with open(main_fpath) as f:
            content = yaml.load(f)
        list_fpath=[]
        if "import" in content.keys():
            list_fpath = content['import'].values()
                
        for fname in list_fpath:
            fpath = os.path.join(simulation_dir, fname)
            with open(fpath) as f:
                content1 = yaml.load(f)
                if "globals" in content1: 
                    if "globals" in content: 
                        raise Exception("globals can be defined only once")
                    else : content['globals']=content1['globals']
                if "entities" in content1:
                    if "entities" in content: 
                        for name1 in content1['entities'].keys(): 
                            if name1 in content['entities'].keys():
                                for name2 in content1['entities'][name1].keys(): 
                                    if name2 in content['entities'][name1].keys():
                                        if name2 in ('processes','macros','links'):
                                            for name3 in content1['entities'][name1][name2].keys(): 
                                                if name3 in content['entities'][name1][name2].keys():
                                                    raise Exception("%s of %s is defined a second time in %s" %(name3,name1,fpath)   )
                                                else : content['entities'][name1][name2][name3]=content1['entities'][name1][name2][name3]
                                        if name2 == 'fields':
                                            raise Exception("fields of entities can be defined only once, it is not the case for %s in %s" %(name1,fpath)   )
                                    else : content['entities'][name1][name2]=content1['entities'][name1][name2]
                            else : content['entities'][name1]=content1['entities'][name1]                        
                    else : content['entities']=content1['entities']
                    
                if "simulation" in content1:
                    if "simulation" in content: 
                        raise Exception("simualtion can be defined only once")                        
                    else : content['simulation']=content1['simulation']
                
                
        #XXX: use validictory instead of my custom validator?
        # http://readthedocs.org/docs/validictory/
        validate_dict(content, cls.yaml_layout)

        globals_def = content.get('globals', {})
        periodic_globals = globals_def.get('periodic', [])
        # list of one-item-dicts to list of tuples
        periodic_globals = [d.items()[0] for d in periodic_globals]
        globals_fields = [(name, str_to_type[typestr])
                          for name, typestr in periodic_globals]

        simulation_def = content['simulation']
        seed = simulation_def.get('random_seed')
        if seed is not None:
            seed = int(seed)
            print "using fixed random seed: %d" % seed
            random.seed(seed)
            np.random.seed(seed)

        periods = simulation_def['periods']
        start_period = simulation_def['start_period']
        config.skip_shows = simulation_def.get('skip_shows', False)
        #TODO: check that the value is one of "raise", "skip", "warn"
        config.assertions = simulation_def.get('assertions', 'raise')

        input_def = simulation_def['input']
        input_directory = input_dir if input_dir is not None \
                                    else input_def.get('path', '')
        if not os.path.isabs(input_directory):
            input_directory = os.path.join(simulation_dir, input_directory)
        config.input_directory = input_directory

        output_def = simulation_def['output']
        output_directory = output_dir if output_dir is not None \
                                      else output_def.get('path', '')
        if not os.path.isabs(output_directory):
            output_directory = os.path.join(simulation_dir, output_directory)
        config.output_directory = output_directory

        if output_file is None:
            output_file = output_def['file']
        output_path = os.path.join(output_directory, output_file)

        for k, v in content['entities'].iteritems():
            entity_registry.add(Entity.from_yaml(k, v))

        for entity in entity_registry.itervalues():
            entity.check_links()
            entity.parse_processes(globals_fields)
            entity.compute_lagged_fields()

        init_def = [d.items()[0] for d in simulation_def.get('init', {})]
        init_processes, init_entities = [], set()
        for ent_name, proc_names in init_def:
            if ent_name not in entity_registry:
                raise Exception("Entity '%s' not found" % ent_name)

            entity = entity_registry[ent_name]
            init_entities.add(entity)
            init_processes.extend([(entity.processes[proc_name],1)
                                   for proc_name in proc_names])

        processes_def = [d.items()[0] for d in simulation_def['processes']]
        processes, entities = [], set()
        for ent_name, proc_defs in processes_def:
            entity = entity_registry[ent_name]
            entities.add(entity)
            for proc_def in proc_defs:           
                # proc_def is simply a process name
                if isinstance(proc_def, basestring):
                    # use the default periodicity of 1
                    proc_name, periodicity = proc_def, 1
                else:
                    proc_name, periodicity = proc_def
                processes.append((entity.processes[proc_name], periodicity))

        method = input_def.get('method', 'h5')

        if method == 'h5':
            if input_file is None:
                input_file = input_def['file']
            input_path = os.path.join(input_directory, input_file)
            data_source = H5Data(input_path, output_path)
        elif method == 'void':
            input_path = None
            data_source = Void(output_path)
        else:
            print method, type(method)

        default_entity = simulation_def.get('default_entity')
        return Simulation(globals_fields, periods, start_period,
                          init_processes, init_entities, processes, entities,
                          data_source, default_entity)
Example #4
0
    def from_yaml(cls, main_fpath, input_dir=None, input_file=None, output_dir=None, output_file=None):
        simulation_path = os.path.abspath(main_fpath)
        simulation_dir = os.path.dirname(simulation_path)

        # TODO : add an instruction in yaml_layout in so as to call an other YAML file and use it instead of a list.
        with open(main_fpath) as f:
            content = yaml.load(f)
        list_fpath = []
        if "import" in content.keys():
            list_fpath = content["import"].values()

        for fname in list_fpath:
            fpath = os.path.join(simulation_dir, fname)
            with open(fpath) as f:
                content1 = yaml.load(f)
                if "globals" in content1:
                    if "globals" in content:
                        raise Exception("globals can be defined only once")
                    else:
                        content["globals"] = content1["globals"]
                if "entities" in content1:
                    if "entities" in content:
                        for name1 in content1["entities"].keys():
                            if name1 in content["entities"].keys():
                                for name2 in content1["entities"][name1].keys():
                                    if name2 in content["entities"][name1].keys():
                                        if name2 in ("processes", "macros", "links"):
                                            for name3 in content1["entities"][name1][name2].keys():
                                                if name3 in content["entities"][name1][name2].keys():
                                                    raise Exception(
                                                        "%s of %s is defined a second time in %s"
                                                        % (name3, name1, fpath)
                                                    )
                                                else:
                                                    content["entities"][name1][name2][name3] = content1["entities"][
                                                        name1
                                                    ][name2][name3]
                                        if name2 == "fields":
                                            raise Exception(
                                                "fields of entities can be defined only once, it is not the case for %s in %s"
                                                % (name1, fpath)
                                            )
                                    else:
                                        content["entities"][name1][name2] = content1["entities"][name1][name2]
                            else:
                                content["entities"][name1] = content1["entities"][name1]
                    else:
                        content["entities"] = content1["entities"]

                if "simulation" in content1:
                    if "simulation" in content:
                        raise Exception("simualtion can be defined only once")
                    else:
                        content["simulation"] = content1["simulation"]
        validate_dict(content, cls.yaml_layout)

        # the goal is to get something like:
        # globals_def = {'periodic': [('a': int), ...],
        #                'MIG': int}
        globals_def = {}
        for k, v in content.get("globals", {}).iteritems():
            # periodic is a special case
            if k == "periodic":
                type_ = fields_yaml_to_type(v)
            else:
                # "fields" and "type" are synonyms
                type_def = v.get("fields") or v.get("type")
                if isinstance(type_def, basestring):
                    type_ = field_str_to_type(type_def, "array '%s'" % k)
                else:
                    if not isinstance(type_def, list):
                        raise SyntaxError("invalid structure for globals")
                    type_ = fields_yaml_to_type(type_def)
            globals_def[k] = type_

        simulation_def = content["simulation"]
        seed = simulation_def.get("random_seed")
        if seed is not None:
            seed = int(seed)
            print "using fixed random seed: %d" % seed
            random.seed(seed)
            np.random.seed(seed)

        periods = simulation_def["periods"]
        start_period = simulation_def["start_period"]
        config.skip_shows = simulation_def.get("skip_shows", False)
        # TODO: check that the value is one of "raise", "skip", "warn"
        config.assertions = simulation_def.get("assertions", "raise")

        input_def = simulation_def["input"]
        input_directory = input_dir if input_dir is not None else input_def.get("path", "")
        if not os.path.isabs(input_directory):
            input_directory = os.path.join(simulation_dir, input_directory)
        config.input_directory = input_directory

        output_def = simulation_def["output"]
        output_directory = output_dir if output_dir is not None else output_def.get("path", "")
        if not os.path.isabs(output_directory):
            output_directory = os.path.join(simulation_dir, output_directory)
        config.output_directory = output_directory

        if output_file is None:
            output_file = output_def["file"]
        output_path = os.path.join(output_directory, output_file)

        for k, v in content["entities"].iteritems():
            entity_registry.add(Entity.from_yaml(k, v))

        for entity in entity_registry.itervalues():
            entity.check_links()
            entity.parse_processes(globals_def)
            entity.compute_lagged_fields()

        init_def = [d.items()[0] for d in simulation_def.get("init", {})]
        init_processes, init_entities = [], set()
        for ent_name, proc_names in init_def:
            if ent_name not in entity_registry:
                raise Exception("Entity '%s' not found" % ent_name)

            entity = entity_registry[ent_name]
            init_entities.add(entity)
            init_processes.extend([(entity.processes[proc_name], 1) for proc_name in proc_names])

        processes_def = [d.items()[0] for d in simulation_def["processes"]]
        processes, entities = [], set()
        for ent_name, proc_defs in processes_def:
            entity = entity_registry[ent_name]
            entities.add(entity)
            for proc_def in proc_defs:
                # proc_def is simply a process name
                if isinstance(proc_def, basestring):
                    # use the default periodicity of 1
                    proc_name, periodicity = proc_def, 1
                else:
                    proc_name, periodicity = proc_def
                processes.append((entity.processes[proc_name], periodicity))

        method = input_def.get("method", "h5")

        if method == "h5":
            if input_file is None:
                input_file = input_def["file"]
            input_path = os.path.join(input_directory, input_file)
            data_source = H5Data(input_path, output_path)
        elif method == "void":
            input_path = None
            data_source = Void(output_path)
        else:
            print method, type(method)

        default_entity = simulation_def.get("default_entity")
        return Simulation(
            globals_def,
            periods,
            start_period,
            init_processes,
            init_entities,
            processes,
            entities,
            data_source,
            default_entity,
        )
Example #5
0
    def from_yaml(cls, fpath,
                  input_dir=None, input_file=None,
                  output_dir=None, output_file=None):
        simulation_path = os.path.abspath(fpath)
        simulation_dir = os.path.dirname(simulation_path)
        with open(fpath) as f:
            content = yaml.load(f)

        content = handle_imports(content, simulation_dir)
        validate_dict(content, cls.yaml_layout)

        # the goal is to get something like:
        # globals_def = {'periodic': [('a': int), ...],
        #                'MIG': int}
        globals_def = {}
        for k, v in content.get('globals', {}).iteritems():
            # periodic is a special case
            if k == 'periodic':
                type_ = fields_yaml_to_type(v)
            else:
                # "fields" and "type" are synonyms
                type_def = v.get('fields') or v.get('type')
                if isinstance(type_def, basestring):
                    type_ = field_str_to_type(type_def, "array '%s'" % k)
                else:
                    if not isinstance(type_def, list):
                        raise SyntaxError("invalid structure for globals")
                    type_ = fields_yaml_to_type(type_def)
            globals_def[k] = type_

        simulation_def = content['simulation']
        seed = simulation_def.get('random_seed')
        if seed is not None:
            seed = int(seed)
            print("using fixed random seed: %d" % seed)
            random.seed(seed)
            np.random.seed(seed)
            
        periods = simulation_def['periods']
        time_scale = simulation_def.get('time_scale', 'year')
        retro = simulation_def.get('retro', False)
        
        start_period = simulation_def.get('start_period',None)
        init_period = simulation_def.get('init_period',None)
        if start_period is None and init_period is None:
            raise Exception("Either start_period either init_period should be given.")
        if start_period is not None:
            if init_period is not None: 
                raise Exception("Start_period can't be given if init_period is.")
            step = time_period[time_scale]*(1 - 2*(retro))           
            init_period = addmonth(start_period, step)
        
        config.skip_shows = simulation_def.get('skip_shows', False)
        #TODO: check that the value is one of "raise", "skip", "warn"
        config.assertions = simulation_def.get('assertions', 'raise')
        config.show_timings = simulation_def.get('timings', True)
        
        autodump = simulation_def.get('autodump', None)
        if autodump is True:
            autodump = 'autodump.h5'
        if isinstance(autodump, basestring):
            # by default autodump will dump all rows
            autodump = (autodump, None)
        config.autodump = autodump

        autodiff = simulation_def.get('autodiff', None)
        if autodiff is True:
            autodiff = 'autodump.h5'
        if isinstance(autodiff, basestring):
            # by default autodiff will compare all rows
            autodiff = (autodiff, None)
        config.autodiff = autodiff

        legislation = simulation_def.get('legislation', None)
        final_stat = simulation_def.get('final_stat', None)
        
        input_def = simulation_def['input']
        input_directory = input_dir if input_dir is not None \
                                    else input_def.get('path', '')
        if not os.path.isabs(input_directory):
            input_directory = os.path.join(simulation_dir, input_directory)
        config.input_directory = input_directory

        output_def = simulation_def['output']
        output_directory = output_dir if output_dir is not None \
                                      else output_def.get('path', '')
        if not os.path.isabs(output_directory):
            output_directory = os.path.join(simulation_dir, output_directory)
        if not os.path.exists(output_directory):
            print("creating directory: '%s'" % output_directory)
            os.makedirs(output_directory)
        config.output_directory = output_directory

        if output_file is None:
            output_file = output_def['file']
        output_path = os.path.join(output_directory, output_file)
    
        method = input_def.get('method', 'h5')

        #need to be before processes because in case of legislation, we need input_table for now.
        if method == 'h5':
            if input_file is None:
                input_file = input_def['file']
            input_path = os.path.join(input_directory, input_file)
            data_source = H5Data(input_path, output_path)
        elif method == 'void':
            input_path = None
            data_source = Void(output_path)
        else:
            print(method, type(method))    
    
    
    
        for k, v in content['entities'].iteritems():
            entity_registry.add(Entity.from_yaml(k, v))
            
        for entity in entity_registry.itervalues():
            entity.check_links()
            entity.parse_processes(globals_def)
            entity.compute_lagged_fields()
            
        init_def = [d.items()[0] for d in simulation_def.get('init', {})]
        init_processes, init_entities = [], set()
        for ent_name, proc_names in init_def:
            if ent_name != 'legislation':
                if ent_name not in entity_registry:
                    raise Exception("Entity '%s' not found" % ent_name)
                
                entity = entity_registry[ent_name]
                init_entities.add(entity)
                init_processes.extend([(entity.processes[proc_name], 1, 1)
                                       for proc_name in proc_names])
            else: 
#                 proc1 = ExtProcess('liam2of',['simulation',None])
                proc2 = ExtProcess('of_on_liam',['simulation',2009,'period'])
#                 proc3 = ExtProcess('merge_leg',['simulation',data_source.output_path,
#                                                 "C:/Til/output/"+"simul_leg.h5",'period'])
#                 init_processes.append((proc1, 1))
                init_processes.append((proc2, 1, 1))
#                 processes.append((proc3, 1))


        processes_def = [d.items()[0] for d in simulation_def['processes']]
        processes, entity_set = [], set()
        for ent_name, proc_defs in processes_def:
            if ent_name != 'legislation':
                entity = entity_registry[ent_name]
                entity_set.add(entity)
                for proc_def in proc_defs:
                    # proc_def is simply a process name
                    if isinstance(proc_def, basestring):
                        # use the default periodicity of 1
                        proc_name, periodicity, start = proc_def, 1, 1
                    else:
                        if len(proc_def) == 3:
                            proc_name, periodicity, start = proc_def
                        elif len(proc_def) == 2:
                            proc_name, periodicity = proc_def 
                            start = 1
                    processes.append((entity.processes[proc_name], periodicity, start))
            else: 
#                 proc1 = ExtProcess('liam2of',['simulation',None])
                proc2 = ExtProcess('of_on_liam',['simulation',proc_defs[0],'period'])
#                 proc3 = ExtProcess('merge_leg',['simulation',data_source.output_path,
#                                                 "C:/Til/output/"+"simul_leg.h5",'period'])

#                 processes.append((proc1, 1))
                processes.append((proc2, 'year',12))
#                 processes.append((proc3, 1))
        entities = sorted(entity_set, key=lambda e: e.name)

        default_entity = simulation_def.get('default_entity')
        #processes[2][0].subprocesses[0][0]
        return Simulation(globals_def, periods, init_period,
                          init_processes, init_entities, processes, entities,
                          data_source, default_entity, legislation, final_stat, time_scale, retro)