def setUp(self):
hh_link = links.Many2One('household', 'hh_id', 'household')
mother_link = links.Many2One('mother', 'mother_id', 'person')
child_link = links.One2Many('children', 'mother_id', 'person')
persons_link = links.One2Many('persons', 'hh_id', 'person')
dt = np.dtype([('period', int), ('id', int), ('age', int),
('dead', bool), ('mother_id', int), ('hh_id', int)])
#TODO: I can't use an EntityContext with an array containing several periods
# of data
# persons = array([(2000, 0, 53, False, -1, 0),
# (2000, 1, 23, False, 0, 1),
# (2000, 2, 20, False, 0, 2),
# (2000, 3, 43, False, -1, 3),
# (2001, 0, 54, True, -1, 0),
# (2001, 1, 24, False, 0, 1),
# (2001, 2, 21, False, 0, 2),
# (2001, 3, 44, False, -1, 0), # they got married
# (2001, 4, 0, False, 2, 2),
persons = array([(2002, 0, 55, True, -1, 0),
(2002, 1, 25, False, 0, 1),
(2002, 2, 22, False, 0, 2),
(2002, 3, 45, False, -1, 0),
(2002, 4, 1, False, 2, 2)],
dtype=dt)
person = Entity('person',
links={'household': hh_link,
'mother': mother_link,
'children': child_link},
array=persons)
# person = FakeEntity('person',
# links={'household': hh_link,
# 'mother': mother_link,
# 'children': child_link},
# data=persons)
dt = np.dtype([('period', int), ('id', int)])
# households = array([(2000, 0),
# (2000, 1),
# (2000, 2),
# (2000, 3),
#
# (2001, 0),
# (2001, 1),
# (2001, 2),
households = array([(2002, 0),
(2002, 1),
(2002, 2)],
dtype=dt)
# household = FakeEntity('household',
# links={'persons': persons_link},
# data=households)
household = Entity('household',
links={'persons': persons_link},
array=households)
entity_registry.add(person)
entity_registry.add(household)
def from_yaml(cls, fpath,
input_dir=None, input_file=None,
output_dir=None, output_file=None):
simulation_path = os.path.abspath(fpath)
simulation_dir = os.path.dirname(simulation_path)
with open(fpath) as f:
content = yaml.load(f)
content = handle_imports(content, simulation_dir)
validate_dict(content, cls.yaml_layout)
# the goal is to get something like:
# globals_def = {'periodic': [('a': int), ...],
# 'MIG': int}
globals_def = {}
for k, v in content.get('globals', {}).iteritems():
# periodic is a special case
if k == 'periodic':
type_ = fields_yaml_to_type(v)
else:
# "fields" and "type" are synonyms
type_def = v.get('fields') or v.get('type')
if isinstance(type_def, basestring):
type_ = field_str_to_type(type_def, "array '%s'" % k)
else:
if not isinstance(type_def, list):
raise SyntaxError("invalid structure for globals")
type_ = fields_yaml_to_type(type_def)
globals_def[k] = type_
simulation_def = content['simulation']
seed = simulation_def.get('random_seed')
if seed is not None:
seed = int(seed)
print("using fixed random seed: %d" % seed)
random.seed(seed)
np.random.seed(seed)
periods = simulation_def['periods']
start_period = simulation_def['start_period']
config.skip_shows = simulation_def.get('skip_shows', False)
#TODO: check that the value is one of "raise", "skip", "warn"
config.assertions = simulation_def.get('assertions', 'raise')
config.show_timings = simulation_def.get('timings', True)
autodump = simulation_def.get('autodump', None)
if autodump is True:
autodump = 'autodump.h5'
if isinstance(autodump, basestring):
# by default autodump will dump all rows
autodump = (autodump, None)
config.autodump = autodump
autodiff = simulation_def.get('autodiff', None)
if autodiff is True:
autodiff = 'autodump.h5'
if isinstance(autodiff, basestring):
# by default autodiff will compare all rows
autodiff = (autodiff, None)
config.autodiff = autodiff
input_def = simulation_def['input']
input_directory = input_dir if input_dir is not None \
else input_def.get('path', '')
if not os.path.isabs(input_directory):
input_directory = os.path.join(simulation_dir, input_directory)
config.input_directory = input_directory
output_def = simulation_def['output']
output_directory = output_dir if output_dir is not None \
else output_def.get('path', '')
if not os.path.isabs(output_directory):
output_directory = os.path.join(simulation_dir, output_directory)
if not os.path.exists(output_directory):
print("creating directory: '%s'" % output_directory)
os.makedirs(output_directory)
config.output_directory = output_directory
if output_file is None:
output_file = output_def['file']
output_path = os.path.join(output_directory, output_file)
for k, v in content['entities'].iteritems():
entity_registry.add(Entity.from_yaml(k, v))
for entity in entity_registry.itervalues():
entity.check_links()
entity.parse_processes(globals_def)
entity.compute_lagged_fields()
init_def = [d.items()[0] for d in simulation_def.get('init', {})]
init_processes, init_entities = [], set()
for ent_name, proc_names in init_def:
if ent_name not in entity_registry:
raise Exception("Entity '%s' not found" % ent_name)
entity = entity_registry[ent_name]
init_entities.add(entity)
init_processes.extend([(entity.processes[proc_name], 1)
for proc_name in proc_names])
processes_def = [d.items()[0] for d in simulation_def['processes']]
processes, entity_set = [], set()
for ent_name, proc_defs in processes_def:
entity = entity_registry[ent_name]
entity_set.add(entity)
for proc_def in proc_defs:
# proc_def is simply a process name
if isinstance(proc_def, basestring):
# use the default periodicity of 1
proc_name, periodicity = proc_def, 1
else:
proc_name, periodicity = proc_def
processes.append((entity.processes[proc_name], periodicity))
entities = sorted(entity_set, key=lambda e: e.name)
method = input_def.get('method', 'h5')
if method == 'h5':
if input_file is None:
input_file = input_def['file']
input_path = os.path.join(input_directory, input_file)
data_source = H5Data(input_path, output_path)
elif method == 'void':
data_source = Void(output_path)
else:
raise ValueError("'%s' is an invalid value for 'method'. It should "
"be either 'h5' or 'void'")
default_entity = simulation_def.get('default_entity')
return Simulation(globals_def, periods, start_period,
init_processes, init_entities, processes, entities,
data_source, default_entity)
def from_yaml(cls, main_fpath,
input_dir=None, input_file=None,
output_dir=None, output_file=None):
simulation_path = os.path.abspath(main_fpath)
simulation_dir = os.path.dirname(simulation_path)
#TODO : add an instruction in yaml_layout in so as to call an other YAML file and use it instead of a list.
with open(main_fpath) as f:
content = yaml.load(f)
list_fpath=[]
if "import" in content.keys():
list_fpath = content['import'].values()
for fname in list_fpath:
fpath = os.path.join(simulation_dir, fname)
with open(fpath) as f:
content1 = yaml.load(f)
if "globals" in content1:
if "globals" in content:
raise Exception("globals can be defined only once")
else : content['globals']=content1['globals']
if "entities" in content1:
if "entities" in content:
for name1 in content1['entities'].keys():
if name1 in content['entities'].keys():
for name2 in content1['entities'][name1].keys():
if name2 in content['entities'][name1].keys():
if name2 in ('processes','macros','links'):
for name3 in content1['entities'][name1][name2].keys():
if name3 in content['entities'][name1][name2].keys():
raise Exception("%s of %s is defined a second time in %s" %(name3,name1,fpath) )
else : content['entities'][name1][name2][name3]=content1['entities'][name1][name2][name3]
if name2 == 'fields':
raise Exception("fields of entities can be defined only once, it is not the case for %s in %s" %(name1,fpath) )
else : content['entities'][name1][name2]=content1['entities'][name1][name2]
else : content['entities'][name1]=content1['entities'][name1]
else : content['entities']=content1['entities']
if "simulation" in content1:
if "simulation" in content:
raise Exception("simualtion can be defined only once")
else : content['simulation']=content1['simulation']
#XXX: use validictory instead of my custom validator?
# http://readthedocs.org/docs/validictory/
validate_dict(content, cls.yaml_layout)
globals_def = content.get('globals', {})
periodic_globals = globals_def.get('periodic', [])
# list of one-item-dicts to list of tuples
periodic_globals = [d.items()[0] for d in periodic_globals]
globals_fields = [(name, str_to_type[typestr])
for name, typestr in periodic_globals]
simulation_def = content['simulation']
seed = simulation_def.get('random_seed')
if seed is not None:
seed = int(seed)
print "using fixed random seed: %d" % seed
random.seed(seed)
np.random.seed(seed)
periods = simulation_def['periods']
start_period = simulation_def['start_period']
config.skip_shows = simulation_def.get('skip_shows', False)
#TODO: check that the value is one of "raise", "skip", "warn"
config.assertions = simulation_def.get('assertions', 'raise')
input_def = simulation_def['input']
input_directory = input_dir if input_dir is not None \
else input_def.get('path', '')
if not os.path.isabs(input_directory):
input_directory = os.path.join(simulation_dir, input_directory)
config.input_directory = input_directory
output_def = simulation_def['output']
output_directory = output_dir if output_dir is not None \
else output_def.get('path', '')
if not os.path.isabs(output_directory):
output_directory = os.path.join(simulation_dir, output_directory)
config.output_directory = output_directory
if output_file is None:
output_file = output_def['file']
output_path = os.path.join(output_directory, output_file)
for k, v in content['entities'].iteritems():
entity_registry.add(Entity.from_yaml(k, v))
for entity in entity_registry.itervalues():
entity.check_links()
entity.parse_processes(globals_fields)
entity.compute_lagged_fields()
init_def = [d.items()[0] for d in simulation_def.get('init', {})]
init_processes, init_entities = [], set()
for ent_name, proc_names in init_def:
if ent_name not in entity_registry:
raise Exception("Entity '%s' not found" % ent_name)
entity = entity_registry[ent_name]
init_entities.add(entity)
init_processes.extend([(entity.processes[proc_name],1)
for proc_name in proc_names])
processes_def = [d.items()[0] for d in simulation_def['processes']]
processes, entities = [], set()
for ent_name, proc_defs in processes_def:
entity = entity_registry[ent_name]
entities.add(entity)
for proc_def in proc_defs:
# proc_def is simply a process name
if isinstance(proc_def, basestring):
# use the default periodicity of 1
proc_name, periodicity = proc_def, 1
else:
proc_name, periodicity = proc_def
processes.append((entity.processes[proc_name], periodicity))
method = input_def.get('method', 'h5')
if method == 'h5':
if input_file is None:
input_file = input_def['file']
input_path = os.path.join(input_directory, input_file)
data_source = H5Data(input_path, output_path)
elif method == 'void':
input_path = None
data_source = Void(output_path)
else:
print method, type(method)
default_entity = simulation_def.get('default_entity')
return Simulation(globals_fields, periods, start_period,
init_processes, init_entities, processes, entities,
data_source, default_entity)
def from_yaml(cls, main_fpath, input_dir=None, input_file=None, output_dir=None, output_file=None):
simulation_path = os.path.abspath(main_fpath)
simulation_dir = os.path.dirname(simulation_path)
# TODO : add an instruction in yaml_layout in so as to call an other YAML file and use it instead of a list.
with open(main_fpath) as f:
content = yaml.load(f)
list_fpath = []
if "import" in content.keys():
list_fpath = content["import"].values()
for fname in list_fpath:
fpath = os.path.join(simulation_dir, fname)
with open(fpath) as f:
content1 = yaml.load(f)
if "globals" in content1:
if "globals" in content:
raise Exception("globals can be defined only once")
else:
content["globals"] = content1["globals"]
if "entities" in content1:
if "entities" in content:
for name1 in content1["entities"].keys():
if name1 in content["entities"].keys():
for name2 in content1["entities"][name1].keys():
if name2 in content["entities"][name1].keys():
if name2 in ("processes", "macros", "links"):
for name3 in content1["entities"][name1][name2].keys():
if name3 in content["entities"][name1][name2].keys():
raise Exception(
"%s of %s is defined a second time in %s"
% (name3, name1, fpath)
)
else:
content["entities"][name1][name2][name3] = content1["entities"][
name1
][name2][name3]
if name2 == "fields":
raise Exception(
"fields of entities can be defined only once, it is not the case for %s in %s"
% (name1, fpath)
)
else:
content["entities"][name1][name2] = content1["entities"][name1][name2]
else:
content["entities"][name1] = content1["entities"][name1]
else:
content["entities"] = content1["entities"]
if "simulation" in content1:
if "simulation" in content:
raise Exception("simualtion can be defined only once")
else:
content["simulation"] = content1["simulation"]
validate_dict(content, cls.yaml_layout)
# the goal is to get something like:
# globals_def = {'periodic': [('a': int), ...],
# 'MIG': int}
globals_def = {}
for k, v in content.get("globals", {}).iteritems():
# periodic is a special case
if k == "periodic":
type_ = fields_yaml_to_type(v)
else:
# "fields" and "type" are synonyms
type_def = v.get("fields") or v.get("type")
if isinstance(type_def, basestring):
type_ = field_str_to_type(type_def, "array '%s'" % k)
else:
if not isinstance(type_def, list):
raise SyntaxError("invalid structure for globals")
type_ = fields_yaml_to_type(type_def)
globals_def[k] = type_
simulation_def = content["simulation"]
seed = simulation_def.get("random_seed")
if seed is not None:
seed = int(seed)
print "using fixed random seed: %d" % seed
random.seed(seed)
np.random.seed(seed)
periods = simulation_def["periods"]
start_period = simulation_def["start_period"]
config.skip_shows = simulation_def.get("skip_shows", False)
# TODO: check that the value is one of "raise", "skip", "warn"
config.assertions = simulation_def.get("assertions", "raise")
input_def = simulation_def["input"]
input_directory = input_dir if input_dir is not None else input_def.get("path", "")
if not os.path.isabs(input_directory):
input_directory = os.path.join(simulation_dir, input_directory)
config.input_directory = input_directory
output_def = simulation_def["output"]
output_directory = output_dir if output_dir is not None else output_def.get("path", "")
if not os.path.isabs(output_directory):
output_directory = os.path.join(simulation_dir, output_directory)
config.output_directory = output_directory
if output_file is None:
output_file = output_def["file"]
output_path = os.path.join(output_directory, output_file)
for k, v in content["entities"].iteritems():
entity_registry.add(Entity.from_yaml(k, v))
for entity in entity_registry.itervalues():
entity.check_links()
entity.parse_processes(globals_def)
entity.compute_lagged_fields()
init_def = [d.items()[0] for d in simulation_def.get("init", {})]
init_processes, init_entities = [], set()
for ent_name, proc_names in init_def:
if ent_name not in entity_registry:
raise Exception("Entity '%s' not found" % ent_name)
entity = entity_registry[ent_name]
init_entities.add(entity)
init_processes.extend([(entity.processes[proc_name], 1) for proc_name in proc_names])
processes_def = [d.items()[0] for d in simulation_def["processes"]]
processes, entities = [], set()
for ent_name, proc_defs in processes_def:
entity = entity_registry[ent_name]
entities.add(entity)
for proc_def in proc_defs:
# proc_def is simply a process name
if isinstance(proc_def, basestring):
# use the default periodicity of 1
proc_name, periodicity = proc_def, 1
else:
proc_name, periodicity = proc_def
processes.append((entity.processes[proc_name], periodicity))
method = input_def.get("method", "h5")
if method == "h5":
if input_file is None:
input_file = input_def["file"]
input_path = os.path.join(input_directory, input_file)
data_source = H5Data(input_path, output_path)
elif method == "void":
input_path = None
data_source = Void(output_path)
else:
print method, type(method)
default_entity = simulation_def.get("default_entity")
return Simulation(
globals_def,
periods,
start_period,
init_processes,
init_entities,
processes,
entities,
data_source,
default_entity,
)
def from_yaml(cls, fpath,
input_dir=None, input_file=None,
output_dir=None, output_file=None):
simulation_path = os.path.abspath(fpath)
simulation_dir = os.path.dirname(simulation_path)
with open(fpath) as f:
content = yaml.load(f)
content = handle_imports(content, simulation_dir)
validate_dict(content, cls.yaml_layout)
# the goal is to get something like:
# globals_def = {'periodic': [('a': int), ...],
# 'MIG': int}
globals_def = {}
for k, v in content.get('globals', {}).iteritems():
# periodic is a special case
if k == 'periodic':
type_ = fields_yaml_to_type(v)
else:
# "fields" and "type" are synonyms
type_def = v.get('fields') or v.get('type')
if isinstance(type_def, basestring):
type_ = field_str_to_type(type_def, "array '%s'" % k)
else:
if not isinstance(type_def, list):
raise SyntaxError("invalid structure for globals")
type_ = fields_yaml_to_type(type_def)
globals_def[k] = type_
simulation_def = content['simulation']
seed = simulation_def.get('random_seed')
if seed is not None:
seed = int(seed)
print("using fixed random seed: %d" % seed)
random.seed(seed)
np.random.seed(seed)
periods = simulation_def['periods']
time_scale = simulation_def.get('time_scale', 'year')
retro = simulation_def.get('retro', False)
start_period = simulation_def.get('start_period',None)
init_period = simulation_def.get('init_period',None)
if start_period is None and init_period is None:
raise Exception("Either start_period either init_period should be given.")
if start_period is not None:
if init_period is not None:
raise Exception("Start_period can't be given if init_period is.")
step = time_period[time_scale]*(1 - 2*(retro))
init_period = addmonth(start_period, step)
config.skip_shows = simulation_def.get('skip_shows', False)
#TODO: check that the value is one of "raise", "skip", "warn"
config.assertions = simulation_def.get('assertions', 'raise')
config.show_timings = simulation_def.get('timings', True)
autodump = simulation_def.get('autodump', None)
if autodump is True:
autodump = 'autodump.h5'
if isinstance(autodump, basestring):
# by default autodump will dump all rows
autodump = (autodump, None)
config.autodump = autodump
autodiff = simulation_def.get('autodiff', None)
if autodiff is True:
autodiff = 'autodump.h5'
if isinstance(autodiff, basestring):
# by default autodiff will compare all rows
autodiff = (autodiff, None)
config.autodiff = autodiff
legislation = simulation_def.get('legislation', None)
final_stat = simulation_def.get('final_stat', None)
input_def = simulation_def['input']
input_directory = input_dir if input_dir is not None \
else input_def.get('path', '')
if not os.path.isabs(input_directory):
input_directory = os.path.join(simulation_dir, input_directory)
config.input_directory = input_directory
output_def = simulation_def['output']
output_directory = output_dir if output_dir is not None \
else output_def.get('path', '')
if not os.path.isabs(output_directory):
output_directory = os.path.join(simulation_dir, output_directory)
if not os.path.exists(output_directory):
print("creating directory: '%s'" % output_directory)
os.makedirs(output_directory)
config.output_directory = output_directory
if output_file is None:
output_file = output_def['file']
output_path = os.path.join(output_directory, output_file)
method = input_def.get('method', 'h5')
#need to be before processes because in case of legislation, we need input_table for now.
if method == 'h5':
if input_file is None:
input_file = input_def['file']
input_path = os.path.join(input_directory, input_file)
data_source = H5Data(input_path, output_path)
elif method == 'void':
input_path = None
data_source = Void(output_path)
else:
print(method, type(method))
for k, v in content['entities'].iteritems():
entity_registry.add(Entity.from_yaml(k, v))
for entity in entity_registry.itervalues():
entity.check_links()
entity.parse_processes(globals_def)
entity.compute_lagged_fields()
init_def = [d.items()[0] for d in simulation_def.get('init', {})]
init_processes, init_entities = [], set()
for ent_name, proc_names in init_def:
if ent_name != 'legislation':
if ent_name not in entity_registry:
raise Exception("Entity '%s' not found" % ent_name)
entity = entity_registry[ent_name]
init_entities.add(entity)
init_processes.extend([(entity.processes[proc_name], 1, 1)
for proc_name in proc_names])
else:
# proc1 = ExtProcess('liam2of',['simulation',None])
proc2 = ExtProcess('of_on_liam',['simulation',2009,'period'])
# proc3 = ExtProcess('merge_leg',['simulation',data_source.output_path,
# "C:/Til/output/"+"simul_leg.h5",'period'])
# init_processes.append((proc1, 1))
init_processes.append((proc2, 1, 1))
# processes.append((proc3, 1))
processes_def = [d.items()[0] for d in simulation_def['processes']]
processes, entity_set = [], set()
for ent_name, proc_defs in processes_def:
if ent_name != 'legislation':
entity = entity_registry[ent_name]
entity_set.add(entity)
for proc_def in proc_defs:
# proc_def is simply a process name
if isinstance(proc_def, basestring):
# use the default periodicity of 1
proc_name, periodicity, start = proc_def, 1, 1
else:
if len(proc_def) == 3:
proc_name, periodicity, start = proc_def
elif len(proc_def) == 2:
proc_name, periodicity = proc_def
start = 1
processes.append((entity.processes[proc_name], periodicity, start))
else:
# proc1 = ExtProcess('liam2of',['simulation',None])
proc2 = ExtProcess('of_on_liam',['simulation',proc_defs[0],'period'])
# proc3 = ExtProcess('merge_leg',['simulation',data_source.output_path,
# "C:/Til/output/"+"simul_leg.h5",'period'])
# processes.append((proc1, 1))
processes.append((proc2, 'year',12))
# processes.append((proc3, 1))
entities = sorted(entity_set, key=lambda e: e.name)
default_entity = simulation_def.get('default_entity')
#processes[2][0].subprocesses[0][0]
return Simulation(globals_def, periods, init_period,
init_processes, init_entities, processes, entities,
data_source, default_entity, legislation, final_stat, time_scale, retro)
