class PopgenManager(object): """ The class reads the configuration file, creates the component and model objects, and runs the models to simulate the various activity-travel choice processes. If the configObject is invalid, then a valid fileLoc is desired and if that fails as well then an exception is raised. In a commandline implementation, fileLoc will be passed. """ def __init__(self, fileLoc=None, configObject=None, parallelFlag=1): if fileLoc == None and configObject == None: fileLoc = os.path.realpath('./configuration/config.xml') print 'file location is now assigned to default - ', fileLoc if configObject is None and fileLoc is None: raise ConfigurationError, """The configuration input is not valid; a """\ """location of the XML configuration file or a valid etree """\ """object must be passed""" if not ((isinstance(configObject, etree._ElementTree) or isinstance( configObject, etree._Element)) and configObject is not None): print ConfigurationError, """The configuration object input is not a valid """\ """etree.Element object. Trying to load the object from the configuration"""\ """ file.""" self.fileLoc = fileLoc if configObject is None: configObject = etree.parse(fileLoc) self.configObject = configObject self.parallelFlag = parallelFlag print '________________________________________________________________' print 'PARSING CONFIG FILE' self.configParser = ConfigParser( self.configObject) #creates the model configuration parser self.configParser.parse_project() self.project = self.configParser.project self.create_database() print 'COMPLETED PARSING CONFIG FILE AND CREATED THE MASTER DATABASE' print '________________________________________________________________' try: os.mkdir('%s%s%s' % (self.project.location, os.path.sep, self.project.name)) except Exception, e: print('Warning when creating folder:', e) ppservers = () self.job_server = pp.Server(ppservers=ppservers, restart=True)
def resolve_consistency(self, data, seed): print 'Data processing for use in PopGen' popgenConfigObject = ConfigParserPopGen(self.popgenConfig) popgenManagerObj = PopgenManager(configObject=self.popgenConfig) popgenConfigObject.parse_project() popgenConfigObject.parse_scenarios() scenario = popgenConfigObject.scenarioList[0] print 'Household vars - ', self.hhldColNames, len(self.hhldColNames) print 'Extra household vars - ', scenario.hhldVars, len(scenario.hhldVars) hhldVars = copy.deepcopy(self.hhldColNames) for var in scenario.hhldVars: if var not in hhldVars: hhldVars.append(var) print '\nHousehold vars - ', self.personColNames, len(self.personColNames) print 'Extra household vars - ', scenario.personVars, len(scenario.personVars) personVars = copy.deepcopy(self.personColNames) for var in scenario.personVars: if var not in personVars: personVars.append(var) print '\nHousehold sample file location - ', scenario.sampleUserProv.hhLocation print 'Person sample file location - ', scenario.sampleUserProv.personLocation # Create Index Matrix data.sort([self.idSpec.hidName, self.idSpec.pidName]) self.create_indices(data) self.create_col_numbers(data._colnames) #highestHid = amax(self.hhldIndicesOfPersons[:,0]) #fHhld = csv.writer(open(scenario.sampleUserProv.hhLocation, 'w'), delimiter=",") fHhld = open(scenario.sampleUserProv.hhLocation, 'w') hhldVars_forPopGen = copy.deepcopy(hhldVars) try: hidIndex = hhldVars_forPopGen.index('houseid') hhldVars_forPopGen[hidIndex] = 'hhid' except ValueError, e: print 'Error replacing "houseid" name - ', e hidIndex = 0
def resolve_consistency(self, data, seed): print 'Data processing for use in PopGen' popgenConfigObject = ConfigParserPopGen(self.popgenConfig) popgenManagerObj = PopgenManager(configObject=self.popgenConfig) popgenConfigObject.parse_project() popgenConfigObject.parse_scenarios() scenario = popgenConfigObject.scenarioList[0] print 'Household vars - ', self.hhldColNames, len(self.hhldColNames) print 'Extra household vars - ', scenario.hhldVars, len( scenario.hhldVars) hhldVars = copy.deepcopy(self.hhldColNames) for var in scenario.hhldVars: if var not in hhldVars: hhldVars.append(var) print '\nHousehold vars - ', self.personColNames, len( self.personColNames) print 'Extra household vars - ', scenario.personVars, len( scenario.personVars) personVars = copy.deepcopy(self.personColNames) for var in scenario.personVars: if var not in personVars: personVars.append(var) print '\nHousehold sample file location - ', scenario.sampleUserProv.hhLocation print 'Person sample file location - ', scenario.sampleUserProv.personLocation # Create Index Matrix data.sort([self.idSpec.hidName, self.idSpec.pidName]) self.create_indices(data) self.create_col_numbers(data._colnames) #highestHid = amax(self.hhldIndicesOfPersons[:,0]) #fHhld = csv.writer(open(scenario.sampleUserProv.hhLocation, 'w'), delimiter=",") fHhld = open(scenario.sampleUserProv.hhLocation, 'w') hhldVars_forPopGen = copy.deepcopy(hhldVars) try: hidIndex = hhldVars_forPopGen.index('houseid') hhldVars_forPopGen[hidIndex] = 'hhid' except ValueError, e: print 'Error replacing "houseid" name - ', e hidIndex = 0
def __init__(self, fileLoc=None, configObject=None, parallelFlag=1): if fileLoc == None and configObject == None: fileLoc = os.path.realpath('./configuration/config.xml') print 'file location is now assigned to default - ', fileLoc if configObject is None and fileLoc is None: raise ConfigurationError, """The configuration input is not valid; a """\ """location of the XML configuration file or a valid etree """\ """object must be passed""" if not ((isinstance(configObject, etree._ElementTree) or isinstance( configObject, etree._Element)) and configObject is not None): print ConfigurationError, """The configuration object input is not a valid """\ """etree.Element object. Trying to load the object from the configuration"""\ """ file.""" self.fileLoc = fileLoc if configObject is None: configObject = etree.parse(fileLoc) self.configObject = configObject self.parallelFlag = parallelFlag print '________________________________________________________________' print 'PARSING CONFIG FILE' self.configParser = ConfigParser( self.configObject) #creates the model configuration parser self.configParser.parse_project() self.project = self.configParser.project self.create_database() print 'COMPLETED PARSING CONFIG FILE AND CREATED THE MASTER DATABASE' print '________________________________________________________________' try: os.mkdir('%s%s%s' % (self.project.location, os.path.sep, self.project.name)) except Exception, e: print('Warning when creating folder:', e)
def resolve_consistency(self, data, seed): print 'Data processing for use in PopGen' popgenConfigObject = ConfigParserPopGen(self.popgenConfig) popgenManagerObj = PopgenManager(configObject=self.popgenConfig) popgenConfigObject.parse_project() popgenConfigObject.parse_scenarios() scenario = popgenConfigObject.scenarioList[0] print 'Household vars - ', self.hhldColNames, len(self.hhldColNames) print 'Extra household vars - ', scenario.hhldVars, len(scenario.hhldVars) hhldVars = copy.deepcopy(self.hhldColNames) for var in scenario.hhldVars: if var not in hhldVars: hhldVars.append(var) print '\nHousehold vars - ', self.personColNames, len(self.personColNames) print 'Extra household vars - ', scenario.personVars, len(scenario.personVars) personVars = copy.deepcopy(self.personColNames) for var in scenario.personVars: if var not in personVars: personVars.append(var) print '\nHousehold sample file location - ', scenario.sampleUserProv.hhLocation print 'Person sample file location - ', scenario.sampleUserProv.personLocation # Create Index Matrix data.sort([self.idSpec.hidName, self.idSpec.pidName]) self.create_indices(data) self.create_col_numbers(data._colnames) highestHid = amax(self.hhldIndicesOfPersons[:,0]) #fHhld = csv.writer(open(scenario.sampleUserProv.hhLocation, 'w'), delimiter=",") fHhld = open(scenario.sampleUserProv.hhLocation, 'w') hhldVars_forPopGen = copy.deepcopy(hhldVars) hidIndex = hhldVars_forPopGen.index('houseid') hhldVars_forPopGen[hidIndex] = 'hhid' hhldVars_forPopGen.insert(hidIndex+1,'serialno') hhldVars_forPopGen.insert(0,'pumano') hhldVars_forPopGen.insert(0,'state') print 'new hhldvars', hhldVars_forPopGen #fHhld.writerow(hhldVars_forPopGen) hhldVars_str = '' for var in hhldVars_forPopGen: hhldVars_str += '%s,' %var hhldVars_str = hhldVars_str[:-1] #fHhld.writerow(hhldVars_forPopGen) fHhld.write(hhldVars_str) fHhld.write('\n') personVars_forPopGen = copy.deepcopy(personVars) #fPerson = csv.writer(open(scenario.sampleUserProv.personLocation, 'w'), delimiter=",") fPerson = open(scenario.sampleUserProv.personLocation, 'w') hidIndex = personVars_forPopGen.index('houseid') personVars_forPopGen[hidIndex] = 'hhid' pidIndex = personVars_forPopGen.index('personid') personVars_forPopGen[pidIndex] = 'pnum' personVars_forPopGen.insert(hidIndex+1,'serialno') personVars_forPopGen.insert(0,'pumano') personVars_forPopGen.insert(0,'state') print 'new personvars', personVars_forPopGen personVars_str = '' for var in personVars_forPopGen: personVars_str += '%s,' %var personVars_str = personVars_str[:-1] #fPerson.writerow(personVars_forPopGen) fPerson.write(personVars_str) fPerson.write('\n') hhldCols = data.columns(hhldVars) personCols = data.columns(personVars) ti = time.time() hhldRow_forPopGen = [0] * len(hhldVars_forPopGen) hhldRow_forPopGen[0] = 24 hhldRow_forPopGen[1] = 99 personRow_forPopGen = [0] * len(personVars_forPopGen) personRow_forPopGen[0] = 24 personRow_forPopGen[1] = 99 for hhldIndex in self.hhldIndicesOfPersons: firstPersonRec = hhldIndex[1] lastPersonRec = hhldIndex[2] #personRecs = DataArray(data.data[firstPersonRec:lastPersonRec, :], data.varnames) hhldRow = hhldCols.data[firstPersonRec].astype(int) personRows = personCols.data[firstPersonRec:lastPersonRec].astype(int) #print 'Hhlds\n', hhldRow #print 'Persons\n', personRows # Writing out rows for the household sample file hhldRow_forPopGen[3:] = list(hhldRow) hhldRow_forPopGen[2] = hhldRow_forPopGen[3] #fHhld.writerow(list(hhldRow_forPopGen)) fHhld.write(','.join(map(str, list(hhldRow_forPopGen)))) fHhld.write('\n') # Writing out rows for the person sample file for personRow in personRows: personRow_forPopGen[3:] = personRow personRow_forPopGen[2] = personRow_forPopGen[3] #fPerson.writerow(list(personRow_forPopGen)) fPerson.write(','.join(map(str, list(personRow_forPopGen)))) fPerson.write('\n') fHhld.close() fPerson.close() print '\t Sample input files created in - %.4f' %(time.time()-ti) print 'Startint to synthesize population - ' ti = time.time() popgenManagerObj = PopgenManager(configObject=self.popgenConfig) popgenManagerObj.run_scenarios() print '\t Synthesis complete in - %.4f' %(time.time()-ti) hhldSynLoc = scenario.synHousingTableNameLoc.location hhldSynFileName = scenario.synHousingTableNameLoc.name fHhldSyn = csv.reader(open('%s%s%s.dat' %(hhldSynLoc, os.path.sep, hhldSynFileName), 'r'), delimiter='\t') hhldSynArr = self.load_file(fHhldSyn) print hhldSynArr[:5,:] hhldSynVars = self.parse_meta_file('%s%s%s_meta.txt' %(hhldSynLoc, os.path.sep, hhldSynFileName)) print 'Old var names - ', hhldSynVars hidIndex_popgen = hhldSynVars.index('hhid') hhldSynVars[hidIndex_popgen] = 'houseid' print 'New var names - ', hhldSynVars persSynLoc = scenario.synPersTableNameLoc.location persSynFileName = scenario.synPersTableNameLoc.name fPersSyn = csv.reader(open('%s%s%s.dat' %(persSynLoc, os.path.sep, persSynFileName), 'r'), delimiter='\t') persSynArr = self.load_file(fPersSyn) print persSynArr[:5,:] personSynVars = self.parse_meta_file('%s%s%s_meta.txt' %(persSynLoc, os.path.sep, persSynFileName)) print 'Old var names - ', personSynVars hidIndex_popgen = personSynVars.index('hhid') personSynVars[hidIndex_popgen] = 'houseid' pidIndex_popgen = personSynVars.index('pnum') personSynVars[pidIndex_popgen] = 'personid' print 'New var names - ', personSynVars print '\nHousehold sample file location - ', scenario.synHousingTableNameLoc print 'Person sample file location - ', scenario.synPersTableNameLoc return DataArray(persSynArr, personSynVars), DataArray(hhldSynArr, hhldSynVars)
def resolve_consistency(self, data, seed): print 'Data processing for use in PopGen' popgenConfigObject = ConfigParserPopGen(self.popgenConfig) popgenManagerObj = PopgenManager(configObject=self.popgenConfig) popgenConfigObject.parse_project() popgenConfigObject.parse_scenarios() scenario = popgenConfigObject.scenarioList[0] print 'Household vars - ', self.hhldColNames, len(self.hhldColNames) print 'Extra household vars - ', scenario.hhldVars, len( scenario.hhldVars) hhldVars = copy.deepcopy(self.hhldColNames) for var in scenario.hhldVars: if var not in hhldVars: hhldVars.append(var) print '\nHousehold vars - ', self.personColNames, len( self.personColNames) print 'Extra household vars - ', scenario.personVars, len( scenario.personVars) personVars = copy.deepcopy(self.personColNames) for var in scenario.personVars: if var not in personVars: personVars.append(var) print '\nHousehold sample file location - ', scenario.sampleUserProv.hhLocation print 'Person sample file location - ', scenario.sampleUserProv.personLocation # Create Index Matrix data.sort([self.idSpec.hidName, self.idSpec.pidName]) self.create_indices(data) self.create_col_numbers(data._colnames) highestHid = amax(self.hhldIndicesOfPersons[:, 0]) #fHhld = csv.writer(open(scenario.sampleUserProv.hhLocation, 'w'), delimiter=",") fHhld = open(scenario.sampleUserProv.hhLocation, 'w') hhldVars_forPopGen = copy.deepcopy(hhldVars) hidIndex = hhldVars_forPopGen.index('houseid') hhldVars_forPopGen[hidIndex] = 'hhid' hhldVars_forPopGen.insert(hidIndex + 1, 'serialno') hhldVars_forPopGen.insert(0, 'pumano') hhldVars_forPopGen.insert(0, 'state') print 'new hhldvars', hhldVars_forPopGen #fHhld.writerow(hhldVars_forPopGen) hhldVars_str = '' for var in hhldVars_forPopGen: hhldVars_str += '%s,' % var hhldVars_str = hhldVars_str[:-1] #fHhld.writerow(hhldVars_forPopGen) fHhld.write(hhldVars_str) fHhld.write('\n') personVars_forPopGen = copy.deepcopy(personVars) #fPerson = csv.writer(open(scenario.sampleUserProv.personLocation, 'w'), delimiter=",") fPerson = open(scenario.sampleUserProv.personLocation, 'w') hidIndex = personVars_forPopGen.index('houseid') personVars_forPopGen[hidIndex] = 'hhid' pidIndex = personVars_forPopGen.index('personid') personVars_forPopGen[pidIndex] = 'pnum' personVars_forPopGen.insert(hidIndex + 1, 'serialno') personVars_forPopGen.insert(0, 'pumano') personVars_forPopGen.insert(0, 'state') print 'new personvars', personVars_forPopGen personVars_str = '' for var in personVars_forPopGen: personVars_str += '%s,' % var personVars_str = personVars_str[:-1] #fPerson.writerow(personVars_forPopGen) fPerson.write(personVars_str) fPerson.write('\n') hhldCols = data.columns(hhldVars) personCols = data.columns(personVars) ti = time.time() hhldRow_forPopGen = [0] * len(hhldVars_forPopGen) hhldRow_forPopGen[0] = 24 hhldRow_forPopGen[1] = 99 personRow_forPopGen = [0] * len(personVars_forPopGen) personRow_forPopGen[0] = 24 personRow_forPopGen[1] = 99 for hhldIndex in self.hhldIndicesOfPersons: firstPersonRec = hhldIndex[1] lastPersonRec = hhldIndex[2] #personRecs = DataArray(data.data[firstPersonRec:lastPersonRec, :], data.varnames) hhldRow = hhldCols.data[firstPersonRec].astype(int) personRows = personCols.data[firstPersonRec:lastPersonRec].astype( int) #print 'Hhlds\n', hhldRow #print 'Persons\n', personRows # Writing out rows for the household sample file hhldRow_forPopGen[3:] = list(hhldRow) hhldRow_forPopGen[2] = hhldRow_forPopGen[3] #fHhld.writerow(list(hhldRow_forPopGen)) fHhld.write(','.join(map(str, list(hhldRow_forPopGen)))) fHhld.write('\n') # Writing out rows for the person sample file for personRow in personRows: personRow_forPopGen[3:] = personRow personRow_forPopGen[2] = personRow_forPopGen[3] #fPerson.writerow(list(personRow_forPopGen)) fPerson.write(','.join(map(str, list(personRow_forPopGen)))) fPerson.write('\n') fHhld.close() fPerson.close() print '\t Sample input files created in - %.4f' % (time.time() - ti) print 'Startint to synthesize population - ' ti = time.time() popgenManagerObj = PopgenManager(configObject=self.popgenConfig) popgenManagerObj.run_scenarios() print '\t Synthesis complete in - %.4f' % (time.time() - ti) hhldSynLoc = scenario.synHousingTableNameLoc.location hhldSynFileName = scenario.synHousingTableNameLoc.name fHhldSyn = csv.reader(open( '%s%s%s.dat' % (hhldSynLoc, os.path.sep, hhldSynFileName), 'r'), delimiter='\t') hhldSynArr = self.load_file(fHhldSyn) print hhldSynArr[:5, :] hhldSynVars = self.parse_meta_file( '%s%s%s_meta.txt' % (hhldSynLoc, os.path.sep, hhldSynFileName)) print 'Old var names - ', hhldSynVars hidIndex_popgen = hhldSynVars.index('hhid') hhldSynVars[hidIndex_popgen] = 'houseid' print 'New var names - ', hhldSynVars persSynLoc = scenario.synPersTableNameLoc.location persSynFileName = scenario.synPersTableNameLoc.name fPersSyn = csv.reader(open( '%s%s%s.dat' % (persSynLoc, os.path.sep, persSynFileName), 'r'), delimiter='\t') persSynArr = self.load_file(fPersSyn) print persSynArr[:5, :] personSynVars = self.parse_meta_file( '%s%s%s_meta.txt' % (persSynLoc, os.path.sep, persSynFileName)) print 'Old var names - ', personSynVars hidIndex_popgen = personSynVars.index('hhid') personSynVars[hidIndex_popgen] = 'houseid' pidIndex_popgen = personSynVars.index('pnum') personSynVars[pidIndex_popgen] = 'personid' print 'New var names - ', personSynVars print '\nHousehold sample file location - ', scenario.synHousingTableNameLoc print 'Person sample file location - ', scenario.synPersTableNameLoc return DataArray(persSynArr, personSynVars), DataArray(hhldSynArr, hhldSynVars)