def main(simulation, annee_leg=None,annee_base=None, output='array'): print "annee base", annee_base name_convertion = {'person':'ind','declar':'foy','menage':'men', 'fam':'fam'} #TODO: test output is either a simulation either a string # if not isinstance(output,SurveySimulation) # #### initialisation, si on veut une grosse table de sortie # for ent in ('ind','men','foy','fam'): # del output_h5[ent] # output_h5[ent]=pd.DataFrame() ### list des variable que l'on veut conserver ### Note plus vraiment utile listkeep = {} listkeep['ind'] = ["salsuperbrut", "cotsoc_noncontrib","cotsal_noncontrib","cotsoc_bar","cotsoc_lib","cotpat_contrib","cotpat_noncontrib","cotsal_contrib","cotsal", "impo","psoc","mini","pfam","logt"] listkeep['men']= ["decile","decile_net", "pauvre60", "revdisp", "revini", "revnet", "typ_men", "uc"] listkeep['fam']= ["aah","caah","aeeh","aefa","af","cf","paje", "al","alf","als","apl","ars","asf", "api","apje","asi","aspa","rmi","rsa","rsa_socle"] listkeep['foy']= ["decote", "irpp", "isf_tot", "avantage_qf"] ## on recupere la liste des annees en entree if annee_base is not None: if isinstance(annee_base,int): annee_base = [annee_base] else: get_years = HDFStore("C:/til/output/to_run_leg.h5") years = [x[-4:] for x in dir(get_years.root) if x[0]!='_' ] get_years.close() country = 'france' for year in annee_base: yr = str(year) deb3 = time.clock() simu = SurveySimulation() simu.set_config(year = year, country = country) # mettre les paramètres de a législation 2009 date_str = str(annee_leg)+ '-01-01' date = dt.datetime.strptime(date_str ,"%Y-%m-%d").date() reader = XmlReader(simu.param_file, date) rootNode = reader.tree simu.P_default = Tree2Object(rootNode, defaut=True) simu.P_default.datesim = date simu.P = Tree2Object(rootNode, defaut=False) simu.P.datesim = date # liam2of.main(simulation, year, ".h5") table = liam2of.main(simulation, year, "table") simu.set_config(survey_filename=table, num_table=3, print_missing=False) tps_charge = time.clock()-deb3 print tps_charge, time.clock() deb_comp = time.clock() simu.compute() tps_comp = time.clock() - deb_comp print "total", time.clock()-deb3 # save results in the simulation or in a hdf5 table. deb_write = time.clock() if output == '.h5': # chemin de sortie output = "C:/til/output/" output_h5 = tables.openFile(output+"simul_leg.h5",mode='w') output_entities = output_h5.createGroup("/", "entities", "Entities") for ent in ('ind','men','foy',"fam"): # #TODO: gerer un bon keep pour ne pas avoir trop de variable tab = simu.output_table.table3[ent] ### export par table if ent=='ind': ident = ["idmen","quimen","idfam","quifam","idfoy","quifoy","noi"] else: ident = ["idmen","idfam","idfoy"] renam ={} for nom in ident: renam[nom+'_'+ent] = nom tab = tab.rename(columns=renam) tab = tab[listkeep[ent]+ident] tab['period'] = pd.Series(np.ones(len(tab)) * int(year),dtype=int) ident = 'id'+ent if ent=='ind': ident='noi' #on retire les identifiant sauf celui qui deviendra id list_id = ['idmen','idfoy','idfam','id','quifoy','quifam','quimen','noi'] list_id.remove(ident) to_remove = [x for x in tab.columns if x in list_id] #on n4oublie pas de garder periode tab = tab.drop(to_remove,axis=1) tab = tab.rename(columns={ident:'id'}) tab['id'] = tab['id'].astype(int) name_convertion = {'ind':'person','foy':'declar','men':'menage', 'fam':'famille'} nom = name_convertion[ent] output_type = tab.to_records(index=False).dtype #TODO: ameliorer pour optimiser tout ca, les type to_int = ['id','period'] for x in output_type.descr: if x[0] in to_int : x=(x[0],'<i4') if output_entities.__contains__(nom): output_table = getattr(output_entities, nom) else: output_table = output_h5.createTable('/entities',nom,output_type) output_table.append(tab.to_records(index=False)) output_table.flush() output_h5.close() else: entities = simulation.entities for entity in entities: nom = entity.name if nom in name_convertion: ent = name_convertion[nom] vars = [x for x in simu.output_table.table3[ent].columns if x in entity.array.columns] for var in vars: value = simu.output_table.table3[ent][var] #TODO: test the type if len(entity.array[var]) != len(value): print ent, nom, var, len(entity.array[var]), len(value) pdb.set_trace() entity.array[var] = np.array(value) #TODO: change that ad hoc solution if nom == 'menage': ent = 'fam' vars = [x for x in simu.output_table.table3[ent].columns if x in entity.array.columns] for var in vars: value = simu.output_table.table3[ent][var] #TODO: test the type if len(entity.array[var]) != len(value): print ent, nom, var, len(entity.array[var]), len(value) pdb.set_trace() entity.array[var] = np.array(value) tps_write = time.clock() - deb_write del simu gc.collect() fin3 = time.clock() print ("La législation sur l'année %s vient d'être calculée en %d secondes" " dont %d pour le chargement, %d pour la simul pure et %d pour la sauvegarde") %(year, fin3-deb3, tps_charge, tps_comp, tps_write )
# len(np.unique(tab['idfam'])) # list_qui = tab['idfam'] # double = list_qui.value_counts()[list_qui.value_counts()>1] # tabind = table['ind'].ix[table['ind']['period']==year] store.close() goal.close() # on fais maintenant tourner le modèle OF country = 'france' for year in years: yr = str(year) deb3 = time.clock() simu = SurveySimulation() simu.set_config(year = year, country = country) # mettre les paramètres de a législation 2009 date_str = str(2009)+ '-01-01' date = dt.datetime.strptime(date_str ,"%Y-%m-%d").date() reader = XmlReader(simu.param_file, date) rootNode = reader.tree simu.P_default = Tree2Object(rootNode, defaut=True) simu.P_default.datesim = date simu.P = Tree2Object(rootNode, defaut=False) simu.P.datesim = date simu.set_survey(filename="C:/openfisca/src/countries/france/data/surveyLiam.h5", num_table=3, print_missing=True) simu.compute() for ent in ('ind','men','foy','fam'):