def __init__(self, filename): parser = TrainingDataParser(filename) data = np.array(parser.data, dtype=object) self.__M_data = data[ data[:, 0] == 'M' ] self.__B_data = data[ data[:, 0] == 'B' ] self.__M_data = self.__M_data[:, 1:] self.__B_data = self.__B_data[:, 1:] self.__M_data = self.__M_data.astype(float) self.__B_data = self.__B_data.astype(float) self.sample_count = data.shape[0] self.X = data[:, 1:] self.X = self.X.astype(float) Y_literal = data[:, 0] Y_literal = Y_literal.reshape(self.sample_count, 1) self.Y = Y_literal == 'M' self.Y = self.Y.astype(float) self.__all_descriptions = [] for column in self.X.T: self.__all_descriptions.append(Description(column)) self.__M_descriptions = [] for column in self.__M_data.T: self.__M_descriptions.append(Description(column)) self.__B_descriptions = [] for column in self.__B_data.T: self.__B_descriptions.append(Description(column))
def new_description_form():
movie_id = request.form['description_movieid']
description = request.form['description']
print(f"Movie ID: {movie_id}")
new_description = Description()
new_description.description = description
new_description.movieid = movie_id
data.create_description(new_description)
return redirect(url_for('list_all_movies'))
def build_from(package_path,
src_folder,
manifest_path=None,
description_path=None,
files=[],
excludes=[],
storage_factory=ZipFileStorage):
if manifest_path:
with file(manifest_path) as f:
manifest = Manifest()
manifest.load(f)
else:
node = resolve_path(src_folder, MANIFEST_PATH)
if node:
with node.open() as f:
manifest = Manifest()
manifest.load(f)
else:
logger.error('%s: not found' % MANIFEST_PATH)
raise IOError('%s: not found' % MANIFEST_PATH)
if description_path:
with file(description_path) as f:
description = Description.parse(f)
else:
node = resolve_path(src_folder, DESCRIPTION_PATH)
if node:
with node.open() as f:
description = Description.parse(f)
else:
raise IOError('%s: not found' % DESCRIPTION_PATH)
package_path = make_output_path(package_path, description)
package_files = dict()
from itertools import chain
required_files = chain(manifest, description.required_files())
for path in required_files:
node = resolve_path(src_folder, path)
if node is None:
raise IOError('%s: not found' % path)
package_files[path] = node
files = ((path, resolve_path(src_folder, path)) for path in files)
files = expand_folders(files)
files = exclude_files(excludes, files)
package_files.update(files)
return build(package_path,
manifest,
description,
package_files,
storage_factory=storage_factory)
def build_from(package_path,
src_folder,
manifest_path=None,
description_path=None,
files=[],
excludes=[],
storage_factory=ZipFileStorage):
if manifest_path:
with file(manifest_path) as f:
manifest = Manifest()
manifest.load(f)
else:
node = resolve_path(src_folder, MANIFEST_PATH)
if node:
with node.open() as f:
manifest = Manifest()
manifest.load(f)
else:
logger.error('%s: not found' % MANIFEST_PATH)
raise IOError('%s: not found' % MANIFEST_PATH)
if description_path:
with file(description_path) as f:
description = Description.parse(f)
else:
node = resolve_path(src_folder, DESCRIPTION_PATH)
if node:
with node.open() as f:
description = Description.parse(f)
else:
raise IOError('%s: not found' % DESCRIPTION_PATH)
package_path = make_output_path(package_path, description)
package_files = dict()
from itertools import chain
required_files = chain(manifest, description.required_files())
for path in required_files:
node = resolve_path(src_folder, path)
if node is None:
raise IOError('%s: not found' % path)
package_files[path] = node
files = ((path, resolve_path(src_folder, path)) for path in files)
files = expand_folders(files)
files = exclude_files(excludes, files)
package_files.update(files)
return build(package_path, manifest, description, package_files,
storage_factory=storage_factory)
class Client:
"""This is the class you'll probably want to be using. You simply
pass the constructor the url for the service description file and
issue a search and get back results as an iterable Results object.
The neat thing about a Results object is that it will seamlessly
handle fetching more results from the opensearch server when it can...
so you just need to iterate and can let the paging be taken care of
for you.
from opensearch import Client
client = Client(description_url)
results = client.search("computer")
for result in results:
print result.title
"""
def __init__(self, url, agent="python-opensearch <https://github.com/edsu/opensearch>"):
self.agent = agent
self.description = Description(url, self.agent)
def search(self, search_terms, page_size=25):
"""Perform a search and get back a results object
"""
url = self.description.get_best_template()
query = Query(url)
# set up initial values
query.searchTerms = search_terms
query.count = page_size
# run the results
return Results(query, agent=self.agent)
def __init__(self, model_description, survey_data = None, scenario = None):
super(DataTable, self).__init__()
# Init instance attribute
self.description = None
self.scenario = None
self._isPopulated = False
self.col_names = []
self.table = DataFrame()
self.index = {}
self._nrows = 0
self.datesim = CONF.get('simulation', 'datesim')
self.NMEN = CONF.get('simulation', 'nmen')
self.MAXREV = CONF.get('simulation', 'maxrev')
self.XAXIS = CONF.get('simulation', 'xaxis') + 'i'
# Build the description attribute
if type(model_description) == type(ModelDescription):
descr = model_description()
self.description = Description(descr.columns)
else:
raise Exception("model_description should be an ModelDescription inherited class")
self.col_names = self.description.col_names
if (survey_data and scenario):
raise Exception("should provide either survey_data or scenario but not both")
elif survey_data:
self.populate_from_survey_data(survey_data)
elif scenario:
self.populate_from_scenario(scenario)
def show_main():
doc = '''Usage: oxt-pkg-show [options] <package-path>
--help Print this screen.
'''
from docopt import docopt
args = docopt(doc)
logging.basicConfig(level=logging.INFO)
package_path = args['<package-path>']
with open_storage(package_path) as pkg:
with resolve_path(pkg, MANIFEST_PATH).open() as f:
manifest = Manifest()
manifest.load(f)
with resolve_path(pkg, DESCRIPTION_PATH).open() as f:
description = Description.parse(f)
from description import print_human_readable
print_human_readable(description, pkg)
for path in manifest:
item = manifest[path]
print path, item['media-type'],
node = resolve_path(pkg, path)
if node:
print '-- OK'
else:
print '-- MISSING'
def show_main():
doc = '''Usage: oxt-pkg-show [options] <package-path>
--help Print this screen.
'''
from docopt import docopt
args = docopt(doc)
logging.basicConfig(level=logging.INFO)
package_path = args['<package-path>']
with open_storage(package_path) as pkg:
with resolve_path(pkg, MANIFEST_PATH).open() as f:
manifest = Manifest()
manifest.load(f)
with resolve_path(pkg, DESCRIPTION_PATH).open() as f:
description = Description.parse(f)
from description import print_human_readable
print_human_readable(description, pkg)
for path in manifest:
item = manifest[path]
print path, item['media-type'],
node = resolve_path(pkg, path)
if node:
print '-- OK'
else:
print '-- MISSING'
def compute(self, scs, engine_name, **options):
'''
compute hitting sets to the given set of conflict sets with engine_name and options
'''
options['sort_conflict_sets'] = options.get('sort_conflict_sets', True)
return self.compute_with_description(
Description(scs, sort=options['sort_conflict_sets']), engine_name,
**options)
def proposeDescription(self, objekt, type_, descriptor, oldCodelet):
from description import Description
description = Description(objekt, type_, descriptor)
descriptor.buffer = 100.0
urgency = type_.activation
self.newCodelet('description-strength-tester', oldCodelet, urgency,
description)
class Client:
"""This is the class you'll probably want to be using. You simply
pass the constructor the url for the service description file and
issue a search and get back results as an iterable Results object.
The kinda neat thing about a Results object is that it will seamlessly
handle fetching more results from the opensearch server when it can...
so you just need to iterate and can let the paging be taken care of
for you.
>>> from opensearch import Client
>>> client = Client(description_url)
>>> results = client.search("computer")
>>> print results.totalResults
>>> for result in results:
print result.title
If you have additional paramters to pass in the template just use
their name. So for example if the template was
http://example.org/search?q={searchTerms}&key={key}
you could supply key with:
>>> results = client.search("computer", key="abc123")
Additionally if you want to use prefixed parameter names you'll need to use
the double underscore syntax. So for a template like:
http://example.org/search?q={searchTerms}&key={ex:key}
you could supply ex:key with:
>>> results = client.search("computer", ex__key="abc123")
"""
def __init__(self, url, agent="python-opensearch <https://github.com/edsu/opensearch>"):
self.agent = agent
self.description = Description(url, self.agent)
def search(self, search_terms, page_size=25, **kwargs):
"""Perform a search and get back a results object
"""
url = self.description.get_best_template()
query = Query(url)
# set up initial values
query.searchTerms = search_terms
query.count = page_size
# add any additional parameters to the query
for name, value in kwargs.items():
setattr(query, name, value)
# run the results
return Results(query, agent=self.agent)
def on_die(self):
pos = self.owner.get(Position)
corpse = Entity(
Renderable(random.choice(corpse_texes)),
Description('%s\'s corpse' % get_name(self.owner)),
Position(pos.x, pos.y, Position.ORDER_FLOOR + 1),
)
self.owner.level.add_entity(corpse)
self.owner.level.remove_entity(self.owner)
def _add_features(self):
# TODO: factor this out into feature generator
for room in self._layout.rooms:
feature = random.choice([None, 'light', 'fountain', 'library'])
if feature == 'light':
coords = random.sample([
(room.x + 1, room.y + 1),
(room.x + room.grid.size_x - 2, room.y + 1),
(room.x + 1, room.y + room.grid.size_y - 2),
(room.x + room.grid.size_x - 2,
room.y + room.grid.size_y - 2),
], random.randint(1, 4))
for x, y in coords:
self.add_entity(
Entity(Renderable(light_anim, memorable=True),
Blocker(blocks_movement=True),
Description('Light'),
Position(x, y, Position.ORDER_FEATURES)))
elif feature == 'fountain':
self.add_entity(
Entity(
Renderable(fountain_anim, memorable=True),
Blocker(blocks_movement=True), Description('Fountain'),
Position(room.x + room.grid.size_x / 2,
room.y + room.grid.size_y / 2,
Position.ORDER_FEATURES)))
elif feature == 'library':
y = room.y + room.grid.size_y - 1
for x in xrange(room.x + 1, room.x + room.grid.size_x - 1):
if self._layout.grid[x, y] != LayoutGenerator.TILE_WALL:
continue
if x == room.x + 1 and self._layout.grid[
room.x, y - 1] != LayoutGenerator.TILE_WALL:
continue
if x == room.x + room.grid.size_x - 2 and self._layout.grid[
x + 1, y - 1] != LayoutGenerator.TILE_WALL:
continue
self.add_entity(
Entity(
Renderable(random.choice(library_texes),
memorable=True),
Blocker(blocks_movement=True),
Description('Bookshelf'),
Position(x, y - 1, Position.ORDER_FEATURES)))
def get_description(movie_id) -> []:
place_hold = ("This movie does not have a description yet")
sql_statement = f"SELECT * FROM descriptions WHERE movieid LIKE '{movie_id}'"
rows = execute_select(sql_statement)
rollodex = []
for r in rows:
rollodex.append(Description.from_SQLiteRow(r))
if rollodex == 0:
rollodex.append(place_hold)
return rollodex
def init_main():
doc = '''Usage: oxt-pkg-init [options] <package-path>
--help Print this screen.
'''
from docopt import docopt
args = docopt(doc)
logging.basicConfig(level=logging.INFO)
package_path = args['<package-path>']
manifest = Manifest()
description = Description()
with open_storage(package_path, 'w') as stg:
with makedirs_to_file(stg, MANIFEST_PATH).open('w') as f:
manifest.dump(f)
with makedirs_to_file(stg, DESCRIPTION_PATH).open('w') as f:
description.write(f)
def init_main():
doc = '''Usage: oxt-pkg-init [options] <package-path>
--help Print this screen.
'''
from docopt import docopt
args = docopt(doc)
logging.basicConfig(level=logging.INFO)
package_path = args['<package-path>']
manifest = Manifest()
description = Description()
with open_storage(package_path, 'w') as stg:
with makedirs_to_file(stg, MANIFEST_PATH).open('w') as f:
manifest.dump(f)
with makedirs_to_file(stg, DESCRIPTION_PATH).open('w') as f:
description.write(f)
def __init__(self, filename):
parser = TrainingDataParser(filename)
self.__feature_headers = parser.headers[1:]
self.__feature_count = len(self.__feature_headers)
data = np.array(parser.data, dtype=object)
self.__gryffindor_data = data[data[:, 0] == 'Gryffindor']
self.__hufflepuff_data = data[data[:, 0] == 'Hufflepuff']
self.__ravenclaw_data = data[data[:, 0] == 'Ravenclaw']
self.__slytherin_data = data[data[:, 0] == 'Slytherin']
self.__gryffindor_data = self.__gryffindor_data[:, 1:]
self.__hufflepuff_data = self.__hufflepuff_data[:, 1:]
self.__ravenclaw_data = self.__ravenclaw_data[:, 1:]
self.__slytherin_data = self.__slytherin_data[:, 1:]
self.__gryffindor_data = self.__gryffindor_data.astype(float)
self.__hufflepuff_data = self.__hufflepuff_data.astype(float)
self.__ravenclaw_data = self.__ravenclaw_data.astype(float)
self.__slytherin_data = self.__slytherin_data.astype(float)
self.sample_count = data.shape[0]
self.Y_labeled = data[:, 0]
self.Y_labeled = self.Y_labeled.reshape(self.sample_count, 1)
self.X = data[:, 1:]
self.X = self.X.astype(float)
self.all_descriptions = []
for column in self.X.T:
self.all_descriptions.append(Description(column))
self.__gryffindor_descriptions = []
for column in self.__gryffindor_data.T:
self.__gryffindor_descriptions.append(Description(column))
self.__hufflepulff_descriptions = []
for column in self.__hufflepuff_data.T:
self.__hufflepulff_descriptions.append(Description(column))
self.__ravenclaw_descriptions = []
for column in self.__ravenclaw_data.T:
self.__ravenclaw_descriptions.append(Description(column))
self.__slytherin_descriptions = []
for column in self.__slytherin_data.T:
self.__slytherin_descriptions.append(Description(column))
def _process_layout(self):
grid = self._layout.grid
for x in xrange(grid.size_x):
for y in xrange(grid.size_y):
tile = grid[x, y]
if tile in (LayoutGenerator.TILE_DOOR_CLOSED,
LayoutGenerator.TILE_DOOR_OPEN):
self.add_entity(
create_door(x, y,
tile == LayoutGenerator.TILE_DOOR_OPEN))
self.add_entity(
Entity(Description('Floor'), LayoutRenderable(tile),
Position(x, y)))
elif tile == LayoutGenerator.TILE_WALL:
self.add_entity(
Entity(Description('Wall'), Blocker(True, True),
LayoutRenderable(tile),
Position(x, y, Position.ORDER_WALLS)))
elif tile == LayoutGenerator.TILE_FLOOR:
self.add_entity(
Entity(Description('Floor'), LayoutRenderable(tile),
Position(x, y)))
def _add_items(self):
for room in self._layout.rooms:
if random.random() > 0.3:
continue
x = random.randrange(room.x + 1, room.x + room.grid.size_x - 1)
y = random.randrange(room.y + 1, room.y + room.grid.size_y - 1)
if not self.get_movement_blocker(x, y):
self.add_entity(
Entity(
Description('Gold'),
Renderable(random.choice(gold_texes)),
Position(x, y, order=Position.ORDER_ITEMS),
Item('gold', quantity=random.randint(1, 50)),
))
def __init__(self, model_description, survey_data=None, scenario=None):
super(DataTable, self).__init__()
# Init instance attribute
self.description = None
self.scenario = None
self._isPopulated = False
self.col_names = []
self.table = DataFrame()
self.index = {}
self._nrows = 0
self.datesim = CONF.get('simulation', 'datesim')
self.NMEN = CONF.get('simulation', 'nmen')
self.MAXREV = CONF.get('simulation', 'maxrev')
self.XAXIS = CONF.get('simulation', 'xaxis') + 'i'
# Build the description attribute
if type(model_description) == type(ModelDescription):
descr = model_description()
self.description = Description(descr.columns)
else:
raise Exception(
"model_description should be an ModelDescription inherited class"
)
self.col_names = self.description.col_names
if (survey_data and scenario):
raise Exception(
"should provide either survey_data or scenario but not both")
elif survey_data:
self.populate_from_survey_data(survey_data)
elif scenario:
self.populate_from_scenario(scenario)
def compute_trace(self, formula, engine_name, **options):
'''
Like compute_with_description, but here only a LTL formula is given and a
description object is created from it. For the engine this means that a
trace fulfilling this formula should be computed.
'''
randstate = random.getstate()
self.description = Description(None, formula)
if engine_name not in ENGINES:
raise Exception("Unknown Model Checking Engine: %s" % engine_name)
self.engine = ENGINES[engine_name](self.description, options)
self.engine.start()
self.result = self.engine.get_result()
random.setstate(randstate)
return self.result
def create_random_monster(x, y):
name, tex = get_random_monster_params()
monster = Entity(
Actor(80, monster_act),
Position(x, y, Position.ORDER_CREATURES),
Movement(),
Renderable(tex),
Blocker(blocks_movement=True, bump_function=monster_bump),
Health(2),
Fighter(1, 0),
CorpseGenerator(),
InFOV(),
Description(name),
)
return monster
def __init__(self, address):
super().__init__(address)
self.about = About(self)
self.access = Access(self)
self.amcids = Amcids(self)
self.control = Control(self)
self.description = Description(self)
self.diagnostic = Diagnostic(self)
self.functions = Functions(self)
self.move = Move(self)
self.network = Network(self)
self.res = Res(self)
self.rotcomp = Rotcomp(self)
self.rtin = Rtin(self)
self.rtout = Rtout(self)
self.status = Status(self)
self.system_service = System_service(self)
self.update = Update(self)
def check_main():
doc = '''Usage: oxt-pkg-show [options] <package-path>
--help Print this screen.
'''
from docopt import docopt
args = docopt(doc)
logging.basicConfig(level=logging.INFO)
package_path = args['<package-path>']
with open_storage(package_path) as pkg:
with resolve_path(pkg, MANIFEST_PATH).open() as f:
manifest = Manifest()
manifest.load(f)
with resolve_path(pkg, DESCRIPTION_PATH).open() as f:
description = Description.parse(f)
missing = dict()
for path in manifest:
node = resolve_path(pkg, path)
if node is None:
missing[path] = MANIFEST_PATH
for path in description.required_files():
node = resolve_path(pkg, path)
if node is None:
missing[path] = DESCRIPTION_PATH
if missing:
for path in sorted(missing):
referer = missing[path]
logger.error('%s: MISSING (refered in %s)',
path, referer)
raise SystemExit(1)
else:
logger.info('%s: OK, identifier=%s, version=%s', package_path,
description.identifier, description.version)
def check_main():
doc = '''Usage: oxt-pkg-show [options] <package-path>
--help Print this screen.
'''
from docopt import docopt
args = docopt(doc)
logging.basicConfig(level=logging.INFO)
package_path = args['<package-path>']
with open_storage(package_path) as pkg:
with resolve_path(pkg, MANIFEST_PATH).open() as f:
manifest = Manifest()
manifest.load(f)
with resolve_path(pkg, DESCRIPTION_PATH).open() as f:
description = Description.parse(f)
missing = dict()
for path in manifest:
node = resolve_path(pkg, path)
if node is None:
missing[path] = MANIFEST_PATH
for path in description.required_files():
node = resolve_path(pkg, path)
if node is None:
missing[path] = DESCRIPTION_PATH
if missing:
for path in sorted(missing):
referer = missing[path]
logger.error('%s: MISSING (refered in %s)', path, referer)
raise SystemExit(1)
else:
logger.info('%s: OK, identifier=%s, version=%s', package_path,
description.identifier, description.version)
class DataTable(object):
"""
Construct a SystemSf object is a set of Prestation objects
* title [string]
* comment [string]: text shown on the top of the first data item
"""
def __init__(self, model_description, survey_data=None, scenario=None):
super(DataTable, self).__init__()
# Init instance attribute
self.description = None
self.scenario = None
self._isPopulated = False
self.col_names = []
self.table = DataFrame()
self.index = {}
self._nrows = 0
self.datesim = CONF.get('simulation', 'datesim')
self.NMEN = CONF.get('simulation', 'nmen')
self.MAXREV = CONF.get('simulation', 'maxrev')
self.XAXIS = CONF.get('simulation', 'xaxis') + 'i'
# Build the description attribute
if type(model_description) == type(ModelDescription):
descr = model_description()
self.description = Description(descr.columns)
else:
raise Exception(
"model_description should be an ModelDescription inherited class"
)
self.col_names = self.description.col_names
if (survey_data and scenario):
raise Exception(
"should provide either survey_data or scenario but not both")
elif survey_data:
self.populate_from_survey_data(survey_data)
elif scenario:
self.populate_from_scenario(scenario)
def gen_index(self, units):
self.index = {
'ind': {
0: {
'idxIndi': np.arange(self._nrows),
'idxUnit': np.arange(self._nrows)
},
'nb': self._nrows
},
'noi': {}
}
dct = self.index['noi']
nois = self.table.noi.values
listnoi = np.unique(nois)
for noi in listnoi:
idxIndi = np.sort(np.squeeze((np.argwhere(nois == noi))))
idxUnit = np.searchsorted(listnoi, nois[idxIndi])
temp = {'idxIndi': idxIndi, 'idxUnit': idxUnit}
dct.update({noi: temp})
for unit in units:
enum = self.description.get_col('qui' + unit).enum
try:
idx = getattr(self.table, 'id' + unit).values
qui = getattr(self.table, 'qui' + unit).values
enum = self.description.get_col('qui' + unit).enum
except:
raise Exception(
'DataTable needs columns %s and %s to build index with unit %s'
% ('id' + unit, 'qui' + unit, unit))
self.index[unit] = {}
dct = self.index[unit]
idxlist = np.unique(idx)
dct['nb'] = len(idxlist)
for full, person in enum:
idxIndi = np.sort(np.squeeze((np.argwhere(qui == person))))
# if (person == 0) and (dct['nb'] != len(idxIndi)):
# raise Exception('Invalid index for %s: There is %i %s and %i %s' %(unit, dct['nb'], unit, len(idxIndi), full))
idxUnit = np.searchsorted(idxlist, idx[idxIndi])
temp = {'idxIndi': idxIndi, 'idxUnit': idxUnit}
dct.update({person: temp})
def propagate_to_members(self, unit, col):
'''
Set the variable of all unit member to the value of the (head of) unit
'''
index = self.index[unit]
value = self.get_value(col, index)
enum = self.description.get_col('qui' + unit).enum
for member in enum:
self.set_value(col, value, index, opt=member[1])
def inflate(self, totals):
for varname in totals:
if varname in self.table:
x = sum(
self.table[varname] * self.table['wprm']) / totals[varname]
if x > 0:
self.table[varname] = self.table[varname] / x
def populate_from_survey_data(self, fname):
with open(fname) as survey_data_file:
self.table = read_csv(survey_data_file)
self._nrows = self.table.shape[0]
missing_col = []
for col in self.description.columns.itervalues():
if not col.name in self.table:
missing_col.append(col.name)
self.table[col.name] = col._default
self.table[col.name] = self.table[col.name].astype(col._dtype)
if missing_col:
message = "%i input variables missing\n" % len(missing_col)
for var in missing_col:
message += ' - ' + var + '\n'
print Warning(message)
for var in INDEX:
if ('id' + var) in missing_col:
raise Exception('Survey data needs variable %s' % ('id' + var))
if ('qui' + var) in missing_col:
raise Exception('Survey data needs variable %s' %
('qui' + var))
self.gen_index(INDEX)
self._isPopulated = True
self.set_value('wprm_init', self.get_value('wprm'), self.index['ind'])
# self.calage()
# def calage(self):
#
# # inflate revenues on totals
# fname = os.path.join(data_dir, 'calage.csv')
# f_tot = open(fname)
# totals = read_csv(f_tot,index_col = 0)
# totals = totals[year]
# f_tot.close()
#
# self.inflate(totals)
def populate_from_scenario(self, scenario):
self._nrows = self.NMEN * len(scenario.indiv)
MAXREV = self.MAXREV
datesim = self.datesim
self.table = DataFrame()
idmen = np.arange(60001, 60001 + self.NMEN)
for noi, dct in scenario.indiv.iteritems():
birth = dct['birth']
age = datesim.year - birth.year
agem = 12 * (datesim.year -
birth.year) + datesim.month - birth.month
noidec = dct['noidec']
noichef = dct['noichef']
quifoy = self.description.get_col('quifoy').enum[dct['quifoy']]
quifam = self.description.get_col('quifam').enum[dct['quifam']]
quimen = self.description.get_col('quimen').enum[dct['quimen']]
dct = {
'noi': noi * np.ones(self.NMEN),
'age': age * np.ones(self.NMEN),
'agem': agem * np.ones(self.NMEN),
'quimen': quimen * np.ones(self.NMEN),
'quifoy': quifoy * np.ones(self.NMEN),
'quifam': quifam * np.ones(self.NMEN),
'idmen': idmen,
'idfoy': idmen * 100 + noidec,
'idfam': idmen * 100 + noichef
}
self.table = concat([self.table, DataFrame(dct)],
ignore_index=True)
self.gen_index(INDEX)
for name in self.col_names:
if not name in self.table:
self.table[name] = self.description.get_col(name)._default
index = self.index['men']
nb = index['nb']
for noi, dct in scenario.indiv.iteritems():
for var, val in dct.iteritems():
if var in ('birth', 'noipref', 'noidec', 'noichef', 'quifoy',
'quimen', 'quifam'):
continue
if not index[noi] is None:
self.set_value(var, np.ones(nb) * val, index, noi)
index = self.index['foy']
nb = index['nb']
for noi, dct in scenario.declar.iteritems():
for var, val in dct.iteritems():
if not index[noi] is None:
self.set_value(var, np.ones(nb) * val, index, noi)
index = self.index['men']
nb = index['nb']
for noi, dct in scenario.menage.iteritems():
for var, val in dct.iteritems():
if not index[noi] is None:
self.set_value(var, np.ones(nb) * val, index, noi)
# set xaxis
# TODO: how to set xaxis vals properly
if self.NMEN > 1:
var = self.XAXIS
vls = np.linspace(0, MAXREV, self.NMEN)
self.set_value(
var, vls, {
0: {
'idxIndi': index[0]['idxIndi'],
'idxUnit': index[0]['idxIndi']
}
})
self._isPopulated = True
def get_value(self, varname, index=None, opt=None, sum_=False):
'''
method to read the value in an array
index is a dict with the coordinates of each person in the array
- if index is none, returns the whole column (every person)
- if index is not none, return an array of length len(unit)
opt is a dict with the id of the person for which you want the value
- if opt is None, returns the value for the person 0 (i.e. 'vous' for 'foy', 'chef' for 'fam', 'pref' for 'men')
- if opt is not None, return a dict with key 'person' and values for this person
'''
col = self.description.get_col(varname)
dflt = col._default
dtyp = col._dtype
var = np.array(self.table[varname].values, dtype=col._dtype)
if index is None:
return var
nb = index['nb']
if opt is None:
temp = np.ones(nb, dtype=dtyp) * dflt
idx = index[0]
temp[idx['idxUnit']] = var[idx['idxIndi']]
return temp
else:
out = {}
for person in opt:
temp = np.ones(nb, dtype=dtyp) * dflt
idx = index[person]
temp[idx['idxUnit']] = var[idx['idxIndi']]
out[person] = temp
if sum_ is False:
return out
else:
sumout = 0
for val in out.itervalues():
sumout += val
return sumout
def set_value(self, varname, value, index, opt=None):
if opt is None:
idx = index[0]
else:
idx = index[opt]
# this command should work on later pandas version...
# self.table.ix[idx['idxIndi'], [varname]] = value
# for now, we're doing it manually
col = self.description.get_col(varname)
values = self.table[varname].values
dtyp = col._dtype
temp = np.array(value, dtype=dtyp)
var = np.array(values, dtype=dtyp)
var[idx['idxIndi']] = temp[idx['idxUnit']]
self.table[varname] = var
def to_csv(self, fname):
self.table.to_csv(fname)
def __str__(self):
return self.table.__str__()
def __init__(self, url, agent="python-opensearch <https://github.com/edsu/opensearch>"):
self.agent = agent
self.description = Description(url, self.agent)
def __init__(self, string, groupCategory, directionCategory, facet,
objectList, bondList):
# pylint: disable=too-many-arguments
WorkspaceObject.__init__(self, string)
self.groupCategory = groupCategory
self.directionCategory = directionCategory
self.facet = facet
self.objectList = objectList
self.bondList = bondList
self.bondCategory = self.groupCategory.getRelatedNode(
slipnet.bondCategory)
leftObject = objectList[0]
rightObject = objectList[-1]
self.leftIndex = leftObject.leftIndex
self.leftmost = self.leftIndex == 1
self.rightIndex = rightObject.rightIndex
self.rightmost = self.rightIndex == len(self.string)
self.descriptions = []
self.bondDescriptions = []
self.extrinsicDescriptions = []
self.outgoingBonds = []
self.incomingBonds = []
self.bonds = []
self.leftBond = None
self.rightBond = None
self.correspondence = None
self.changed = False
self.newAnswerLetter = False
self.clampSalience = False
self.name = ''
from description import Description
if self.bondList and len(self.bondList):
firstFacet = self.bondList[0].facet
self.addBondDescription(
Description(self, slipnet.bondFacet, firstFacet))
self.addBondDescription(
Description(self, slipnet.bondCategory, self.bondCategory))
self.addDescription(slipnet.objectCategory, slipnet.group)
self.addDescription(slipnet.groupCategory, self.groupCategory)
if not self.directionCategory:
# sameness group - find letterCategory
letter = self.objectList[0].getDescriptor(self.facet)
self.addDescription(self.facet, letter)
if self.directionCategory:
self.addDescription(slipnet.directionCategory,
self.directionCategory)
if self.spansString():
self.addDescription(slipnet.stringPositionCategory, slipnet.whole)
elif self.leftIndex == 1:
self.addDescription(slipnet.stringPositionCategory,
slipnet.leftmost)
elif self.rightIndex == self.string.length:
self.addDescription(slipnet.stringPositionCategory,
slipnet.rightmost)
elif self.middleObject():
self.addDescription(slipnet.stringPositionCategory, slipnet.middle)
self.add_length_description_category()
def get_desc_data(data):
description_obj = Description(data[0]['petition']['description'])
return description_obj
def __init__(self, url, agent="python-opensearch <https://github.com/edsu/opensearch>"):
self.agent = agent
self.description = Description(url, self.agent)
def __init__(self, string, groupCategory, directionCategory, facet,
objectList, bondList):
WorkspaceObject.__init__(self, string)
self.groupCategory = groupCategory
self.directionCategory = directionCategory
self.facet = facet
self.objectList = objectList
self.bondList = bondList
self.bondCategory = self.groupCategory.getRelatedNode(
slipnet.bondCategory)
leftObject = objectList[0]
rightObject = objectList[-1]
self.leftStringPosition = leftObject.leftStringPosition
self.leftmost = self.leftStringPosition == 1
self.rightStringPosition = rightObject.rightStringPosition
self.rightmost = self.rightStringPosition == len(self.string)
self.descriptions = []
self.bondDescriptions = []
self.extrinsicDescriptions = []
self.outgoingBonds = []
self.incomingBonds = []
self.bonds = []
self.leftBond = None
self.rightBond = None
self.correspondence = None
self.changed = False
self.newAnswerLetter = False
self.clampSalience = False
self.name = ''
from description import Description
if self.bondList and len(self.bondList):
firstFacet = self.bondList[0].facet
self.addBondDescription(
Description(self, slipnet.bondFacet, firstFacet))
self.addBondDescription(
Description(self, slipnet.bondCategory, self.bondCategory))
self.addDescription(slipnet.objectCategory, slipnet.group)
self.addDescription(slipnet.groupCategory, self.groupCategory)
if not self.directionCategory:
# sameness group - find letterCategory
letter = self.objectList[0].getDescriptor(self.facet)
self.addDescription(self.facet, letter)
if self.directionCategory:
self.addDescription(slipnet.directionCategory,
self.directionCategory)
if self.spansString():
self.addDescription(slipnet.stringPositionCategory, slipnet.whole)
elif self.leftStringPosition == 1:
self.addDescription(slipnet.stringPositionCategory,
slipnet.leftmost)
elif self.rightStringPosition == self.string.length:
self.addDescription(slipnet.stringPositionCategory,
slipnet.rightmost)
elif self.middleObject():
self.addDescription(slipnet.stringPositionCategory, slipnet.middle)
#check whether or not to add length description category
probability = self.lengthDescriptionProbability()
if random.random() < probability:
length = len(self.objectList)
if length < 6:
self.addDescription(slipnet.length,
slipnet.numbers[length - 1])
class DataTable(object):
"""
Construct a SystemSf object is a set of Prestation objects
* title [string]
* comment [string]: text shown on the top of the first data item
"""
def __init__(self, model_description, survey_data = None, scenario = None):
super(DataTable, self).__init__()
# Init instance attribute
self.description = None
self.scenario = None
self._isPopulated = False
self.col_names = []
self.table = DataFrame()
self.index = {}
self._nrows = 0
self.datesim = CONF.get('simulation', 'datesim')
self.NMEN = CONF.get('simulation', 'nmen')
self.MAXREV = CONF.get('simulation', 'maxrev')
self.XAXIS = CONF.get('simulation', 'xaxis') + 'i'
# Build the description attribute
if type(model_description) == type(ModelDescription):
descr = model_description()
self.description = Description(descr.columns)
else:
raise Exception("model_description should be an ModelDescription inherited class")
self.col_names = self.description.col_names
if (survey_data and scenario):
raise Exception("should provide either survey_data or scenario but not both")
elif survey_data:
self.populate_from_survey_data(survey_data)
elif scenario:
self.populate_from_scenario(scenario)
def gen_index(self, units):
self.index = {'ind': {0: {'idxIndi':np.arange(self._nrows),
'idxUnit':np.arange(self._nrows)},
'nb': self._nrows},
'noi': {}}
dct = self.index['noi']
nois = self.table.noi.values
listnoi = np.unique(nois)
for noi in listnoi:
idxIndi = np.sort(np.squeeze((np.argwhere(nois == noi))))
idxUnit = np.searchsorted(listnoi, nois[idxIndi])
temp = {'idxIndi':idxIndi, 'idxUnit':idxUnit}
dct.update({noi: temp})
for unit in units:
enum = self.description.get_col('qui'+unit).enum
try:
idx = getattr(self.table, 'id'+unit).values
qui = getattr(self.table, 'qui'+unit).values
enum = self.description.get_col('qui'+unit).enum
except:
raise Exception('DataTable needs columns %s and %s to build index with unit %s' %
('id' + unit, 'qui' + unit, unit))
self.index[unit] = {}
dct = self.index[unit]
idxlist = np.unique(idx)
dct['nb'] = len(idxlist)
for full, person in enum:
idxIndi = np.sort(np.squeeze((np.argwhere(qui == person))))
# if (person == 0) and (dct['nb'] != len(idxIndi)):
# raise Exception('Invalid index for %s: There is %i %s and %i %s' %(unit, dct['nb'], unit, len(idxIndi), full))
idxUnit = np.searchsorted(idxlist, idx[idxIndi])
temp = {'idxIndi':idxIndi, 'idxUnit':idxUnit}
dct.update({person: temp})
def propagate_to_members(self, unit = 'men', col = "wprm"):
'''
Set the variable of all unit member to the value of the (head of) unit
'''
index = self.index[unit]
value = self.get_value(col, index)
enum = self.description.get_col('qui'+unit).enum
for member in enum:
self.set_value(col, value, index, opt = member[1])
def inflate(self, totals):
for varname in totals:
if varname in self.table:
x = sum(self.table[varname]*self.table['wprm'])/totals[varname]
if x>0:
self.table[varname] = self.table[varname]/x
def populate_from_survey_data(self, fname):
with open(fname) as survey_data_file:
self.table = read_csv(survey_data_file)
self._nrows = self.table.shape[0]
missing_col = []
for col in self.description.columns.itervalues():
if not col.name in self.table:
missing_col.append(col.name)
self.table[col.name] = col._default
self.table[col.name].astype(col._dtype)
if missing_col:
message = "%i input variables missing\n" % len(missing_col)
for var in missing_col:
message += ' - '+ var + '\n'
print Warning(message)
for var in INDEX:
if ('id' + var) in missing_col:
raise Exception('Survey data needs variable %s' % ('id' + var))
if ('qui' + var) in missing_col:
raise Exception('Survey data needs variable %s' % ('qui' + var))
self.gen_index(INDEX)
self._isPopulated = True
# self.set_zone_apl()
self.set_value('wprm_init', self.get_value('wprm'),self.index['ind'])
# self.calage()
# def set_zone_apl(self):
# data_dir = CONF.get('paths', 'data_dir')
# fname = os.path.join(data_dir, 'zone_apl_imputation_data')
#
# with open(fname, 'rb') as zone_apl_data:
# zone = pickle.load(zone_apl_data)
#
# code_vec = self.get_value('tu99') + 1e1*self.get_value('tau99') + 1e3*self.get_value('reg') + 1e5*self.get_value('pol99')
# zone_apl = self.get_value('zone_apl')
#
# for code in zone.keys():
# if isinstance(zone[code], int):
# zone_apl[code_vec == code] = zone[code]
# else:
# np.random.seed(0)
# prob = np.random.rand(len(zone_apl[code_vec == code]))
# zone_apl[code_vec == code] = 1+ (zone[code][1]>prob) + (zone[code][2]> prob )
# self.set_value('zone_apl',zone_apl,self.index['men'])
# print self.get_value('zone_apl')
# def calage(self):
# data_dir = CONF.get('paths', 'data_dir')
# year = self.datesim.year
# if year <= 2008:
# print 'calage'
# # update weights with calmar (demography)
# fname_men = os.path.join(data_dir, 'calage_men.csv')
# f_tot = open(fname_men)
# totals = read_csv(f_tot,index_col = (0,1))
#
# marges = {}
# for var, mod in totals.index:
# if not marges.has_key(var):
# marges[var] = {}
#
# marges[var][mod] = totals.get_value((var,mod),year)
# f_tot.close()
#
# totalpop = marges.pop('totalpop')[0]
## marges.pop('cstotpragr')
## marges.pop('naf16pr')
## marges.pop('typmen15')
## marges.pop('ddipl')
## marges.pop('ageq')
# marges.pop('act5') # variable la plus problématique
# param ={'use_proportions': True,
# 'method': 'logit', 'lo':.1, 'up': 10,
# 'totalpop' : totalpop,
# 'xtol': 1e-6}
# self.update_weights(marges, param)
#
# #param = {'totalpop': 62000000, 'use_proportions': True}
#
# # inflate revenues on totals
# fname = os.path.join(data_dir, 'calage.csv')
# f_tot = open(fname)
# totals = read_csv(f_tot,index_col = 0)
# totals = totals[year]
# f_tot.close()
#
# self.inflate(totals)
def populate_from_scenario(self, scenario):
self._nrows = self.NMEN*len(scenario.indiv)
MAXREV = self.MAXREV
datesim = self.datesim
self.table = DataFrame()
idmen = np.arange(60001, 60001 + self.NMEN)
for noi, dct in scenario.indiv.iteritems():
birth = dct['birth']
age = datesim.year- birth.year
agem = 12*(datesim.year- birth.year) + datesim.month - birth.month
noidec = dct['noidec']
noichef = dct['noichef']
quifoy = self.description.get_col('quifoy').enum[dct['quifoy']]
quifam = self.description.get_col('quifam').enum[dct['quifam']]
quimen = self.description.get_col('quimen').enum[dct['quimen']]
dct = {'noi': noi*np.ones(self.NMEN),
'age': age*np.ones(self.NMEN),
'agem': agem*np.ones(self.NMEN),
'quimen': quimen*np.ones(self.NMEN),
'quifoy': quifoy*np.ones(self.NMEN),
'quifam': quifam*np.ones(self.NMEN),
'idmen': idmen,
'idfoy': idmen*100 + noidec,
'idfam': idmen*100 + noichef}
self.table = concat([self.table, DataFrame(dct)], ignore_index = True)
self.gen_index(INDEX)
for name in self.col_names:
if not name in self.table:
self.table[name] = self.description.get_col(name)._default
index = self.index['men']
nb = index['nb']
for noi, dct in scenario.indiv.iteritems():
for var, val in dct.iteritems():
if var in ('birth', 'noipref', 'noidec', 'noichef', 'quifoy', 'quimen', 'quifam'): continue
if not index[noi] is None:
self.set_value(var, np.ones(nb)*val, index, noi)
index = self.index['foy']
nb = index['nb']
for noi, dct in scenario.declar.iteritems():
for var, val in dct.iteritems():
if not index[noi] is None:
self.set_value(var, np.ones(nb)*val, index, noi)
index = self.index['men']
nb = index['nb']
for noi, dct in scenario.menage.iteritems():
for var, val in dct.iteritems():
if not index[noi] is None:
self.set_value(var, np.ones(nb)*val, index, noi)
# set xaxis
# TODO: how to set xaxis vals properly
if self.NMEN>1:
var = self.XAXIS
vls = np.linspace(0, MAXREV, self.NMEN)
self.set_value(var, vls, {0:{'idxIndi': index[0]['idxIndi'], 'idxUnit': index[0]['idxIndi']}})
self._isPopulated = True
def get_value(self, varname, index = None, opt = None, sum_ = False):
'''
method to read the value in an array
index is a dict with the coordinates of each person in the array
- if index is none, returns the whole column (every person)
- if index is not none, return an array of length len(unit)
opt is a dict with the id of the person for which you want the value
- if opt is None, returns the value for the person 0 (i.e. 'vous' for 'foy', 'chef' for 'fam', 'pref' for 'men')
- if opt is not None, return a dict with key 'person' and values for this person
'''
col = self.description.get_col(varname)
dflt = col._default
dtyp = col._dtype
var = np.array(self.table[varname].values, dtype = col._dtype)
if index is None:
return var
nb = index['nb']
if opt is None:
temp = np.ones(nb, dtype = dtyp)*dflt
idx = index[0]
temp[idx['idxUnit']] = var[idx['idxIndi']]
return temp
else:
out = {}
for person in opt:
temp = np.ones(nb, dtype = dtyp)*dflt
idx = index[person]
temp[idx['idxUnit']] = var[idx['idxIndi']]
out[person] = temp
if sum_ is False:
return out
else:
sumout = 0
for val in out.itervalues():
sumout += val
return sumout
def set_value(self, varname, value, index, opt = None):
if opt is None:
idx = index[0]
else:
idx = index[opt]
# this command should work on later pandas version...
# self.table.ix[idx['idxIndi'], [varname]] = value
# for now, we're doing it manually
col = self.description.get_col(varname)
values = self.table[varname].values
dtyp = col._dtype
temp = np.array(value, dtype = dtyp)
var = np.array(values, dtype = dtyp)
var[idx['idxIndi']] = temp[idx['idxUnit']]
self.table[varname] = var
def to_csv(self, fname):
self.table.to_csv(fname)
def __str__(self):
return self.table.__str__()
def update_weights(self, marges, param = {}, weights_in='wprm_init', weights_out='wprm', return_margins = False):
data = {weights_in: self.get_value(weights_in, self.index['men'])}
if marges:
for var in marges:
if var in self.col_names:
data[var] = self.get_value(var, self.index['men'])
# else:
# if var != "totalpop":
# data[var] = self.get_value(var, self.index['men'])
try:
val_pondfin, lambdasol, marge_new = calmar(data, marges, param = param, pondini=weights_in)
except:
raise Exception("Calmar error")
return
else:
val_pondfin = data[weights_in]
marge_new = {}
self.set_value(weights_out, val_pondfin, self.index['men'])
self.propagate_to_members( unit='men', col = weights_out)
if return_margins:
return marge_new
def __init__(self, vals={}):
Description.__init__(self, vals)
def addDescription(self, descriptionType, descriptor):
description = Description(self, descriptionType, descriptor)
logging.info("Adding description: %s to %s" % (description, self))
self.descriptions += [description]
def description(self, description):
self._description = Description(description)
def __init__(self, vals={}):
Description.__init__(self, vals)
class DataTable(object):
"""
Construct a SystemSf object is a set of Prestation objects
* title [string]
* comment [string]: text shown on the top of the first data item
"""
def __init__(self, model_description, survey_data = None, scenario = None):
super(DataTable, self).__init__()
# Init instance attribute
self.description = None
self.scenario = None
self._isPopulated = False
self.col_names = []
self.table = DataFrame()
self.index = {}
self._nrows = 0
self.datesim = CONF.get('simulation', 'datesim')
self.NMEN = CONF.get('simulation', 'nmen')
self.MAXREV = CONF.get('simulation', 'maxrev')
self.XAXIS = CONF.get('simulation', 'xaxis') + 'i'
# Build the description attribute
if type(model_description) == type(ModelDescription):
descr = model_description()
self.description = Description(descr.columns)
else:
raise Exception("model_description should be an ModelDescription inherited class")
self.col_names = self.description.col_names
if (survey_data and scenario):
raise Exception("should provide either survey_data or scenario but not both")
elif survey_data:
self.populate_from_survey_data(survey_data)
elif scenario:
self.populate_from_scenario(scenario)
def gen_index(self, units):
self.index = {'ind': {0: {'idxIndi':np.arange(self._nrows),
'idxUnit':np.arange(self._nrows)},
'nb': self._nrows},
'noi': {}}
dct = self.index['noi']
nois = self.table.noi.values
listnoi = np.unique(nois)
for noi in listnoi:
idxIndi = np.sort(np.squeeze((np.argwhere(nois == noi))))
idxUnit = np.searchsorted(listnoi, nois[idxIndi])
temp = {'idxIndi':idxIndi, 'idxUnit':idxUnit}
dct.update({noi: temp})
for unit in units:
enum = self.description.get_col('qui'+unit).enum
try:
idx = getattr(self.table, 'id'+unit).values
qui = getattr(self.table, 'qui'+unit).values
enum = self.description.get_col('qui'+unit).enum
except:
raise Exception('DataTable needs columns %s and %s to build index with unit %s' %
('id' + unit, 'qui' + unit, unit))
self.index[unit] = {}
dct = self.index[unit]
idxlist = np.unique(idx)
dct['nb'] = len(idxlist)
for full, person in enum:
idxIndi = np.sort(np.squeeze((np.argwhere(qui == person))))
# if (person == 0) and (dct['nb'] != len(idxIndi)):
# raise Exception('Invalid index for %s: There is %i %s and %i %s' %(unit, dct['nb'], unit, len(idxIndi), full))
idxUnit = np.searchsorted(idxlist, idx[idxIndi])
temp = {'idxIndi':idxIndi, 'idxUnit':idxUnit}
dct.update({person: temp})
def propagate_to_members(self, unit , col):
'''
Set the variable of all unit member to the value of the (head of) unit
'''
index = self.index[unit]
value = self.get_value(col, index)
enum = self.description.get_col('qui'+unit).enum
for member in enum:
self.set_value(col, value, index, opt = member[1])
def inflate(self, totals):
for varname in totals:
if varname in self.table:
x = sum(self.table[varname]*self.table['wprm'])/totals[varname]
if x>0:
self.table[varname] = self.table[varname]/x
def populate_from_survey_data(self, fname):
with open(fname) as survey_data_file:
self.table = read_csv(survey_data_file)
self._nrows = self.table.shape[0]
missing_col = []
for col in self.description.columns.itervalues():
if not col.name in self.table:
missing_col.append(col.name)
self.table[col.name] = col._default
self.table[col.name] = self.table[col.name].astype(col._dtype)
if missing_col:
message = "%i input variables missing\n" % len(missing_col)
for var in missing_col:
message += ' - '+ var + '\n'
print Warning(message)
for var in INDEX:
if ('id' + var) in missing_col:
raise Exception('Survey data needs variable %s' % ('id' + var))
if ('qui' + var) in missing_col:
raise Exception('Survey data needs variable %s' % ('qui' + var))
self.gen_index(INDEX)
self._isPopulated = True
self.set_value('wprm_init', self.get_value('wprm'),self.index['ind'])
# self.calage()
# def calage(self):
#
# # inflate revenues on totals
# fname = os.path.join(data_dir, 'calage.csv')
# f_tot = open(fname)
# totals = read_csv(f_tot,index_col = 0)
# totals = totals[year]
# f_tot.close()
#
# self.inflate(totals)
def populate_from_scenario(self, scenario):
self._nrows = self.NMEN*len(scenario.indiv)
MAXREV = self.MAXREV
datesim = self.datesim
self.table = DataFrame()
idmen = np.arange(60001, 60001 + self.NMEN)
for noi, dct in scenario.indiv.iteritems():
birth = dct['birth']
age = datesim.year- birth.year
agem = 12*(datesim.year- birth.year) + datesim.month - birth.month
noidec = dct['noidec']
noichef = dct['noichef']
quifoy = self.description.get_col('quifoy').enum[dct['quifoy']]
quifam = self.description.get_col('quifam').enum[dct['quifam']]
quimen = self.description.get_col('quimen').enum[dct['quimen']]
dct = {'noi': noi*np.ones(self.NMEN),
'age': age*np.ones(self.NMEN),
'agem': agem*np.ones(self.NMEN),
'quimen': quimen*np.ones(self.NMEN),
'quifoy': quifoy*np.ones(self.NMEN),
'quifam': quifam*np.ones(self.NMEN),
'idmen': idmen,
'idfoy': idmen*100 + noidec,
'idfam': idmen*100 + noichef}
self.table = concat([self.table, DataFrame(dct)], ignore_index = True)
self.gen_index(INDEX)
for name in self.col_names:
if not name in self.table:
self.table[name] = self.description.get_col(name)._default
index = self.index['men']
nb = index['nb']
for noi, dct in scenario.indiv.iteritems():
for var, val in dct.iteritems():
if var in ('birth', 'noipref', 'noidec', 'noichef', 'quifoy', 'quimen', 'quifam'): continue
if not index[noi] is None:
self.set_value(var, np.ones(nb)*val, index, noi)
index = self.index['foy']
nb = index['nb']
for noi, dct in scenario.declar.iteritems():
for var, val in dct.iteritems():
if not index[noi] is None:
self.set_value(var, np.ones(nb)*val, index, noi)
index = self.index['men']
nb = index['nb']
for noi, dct in scenario.menage.iteritems():
for var, val in dct.iteritems():
if not index[noi] is None:
self.set_value(var, np.ones(nb)*val, index, noi)
# set xaxis
# TODO: how to set xaxis vals properly
if self.NMEN>1:
var = self.XAXIS
vls = np.linspace(0, MAXREV, self.NMEN)
self.set_value(var, vls, {0:{'idxIndi': index[0]['idxIndi'], 'idxUnit': index[0]['idxIndi']}})
self._isPopulated = True
def get_value(self, varname, index = None, opt = None, sum_ = False):
'''
method to read the value in an array
index is a dict with the coordinates of each person in the array
- if index is none, returns the whole column (every person)
- if index is not none, return an array of length len(unit)
opt is a dict with the id of the person for which you want the value
- if opt is None, returns the value for the person 0 (i.e. 'vous' for 'foy', 'chef' for 'fam', 'pref' for 'men')
- if opt is not None, return a dict with key 'person' and values for this person
'''
col = self.description.get_col(varname)
dflt = col._default
dtyp = col._dtype
var = np.array(self.table[varname].values, dtype = col._dtype)
if index is None:
return var
nb = index['nb']
if opt is None:
temp = np.ones(nb, dtype = dtyp)*dflt
idx = index[0]
temp[idx['idxUnit']] = var[idx['idxIndi']]
return temp
else:
out = {}
for person in opt:
temp = np.ones(nb, dtype = dtyp)*dflt
idx = index[person]
temp[idx['idxUnit']] = var[idx['idxIndi']]
out[person] = temp
if sum_ is False:
return out
else:
sumout = 0
for val in out.itervalues():
sumout += val
return sumout
def set_value(self, varname, value, index, opt = None):
if opt is None:
idx = index[0]
else:
idx = index[opt]
# this command should work on later pandas version...
# self.table.ix[idx['idxIndi'], [varname]] = value
# for now, we're doing it manually
col = self.description.get_col(varname)
values = self.table[varname].values
dtyp = col._dtype
temp = np.array(value, dtype = dtyp)
var = np.array(values, dtype = dtyp)
var[idx['idxIndi']] = temp[idx['idxUnit']]
self.table[varname] = var
def to_csv(self, fname):
self.table.to_csv(fname)
def __str__(self):
return self.table.__str__()
