class PopgenManager(object):
"""
The class reads the configuration file, creates the component and model objects,
and runs the models to simulate the various activity-travel choice processes.
If the configObject is invalid, then a valid fileLoc is desired and if that fails
as well then an exception is raised. In a commandline implementation, fileLoc will
be passed.
"""
def __init__(self, fileLoc=None, configObject=None, parallelFlag=1):
if fileLoc == None and configObject == None:
fileLoc = os.path.realpath('./configuration/config.xml')
print 'file location is now assigned to default - ', fileLoc
if configObject is None and fileLoc is None:
raise ConfigurationError, """The configuration input is not valid; a """\
"""location of the XML configuration file or a valid etree """\
"""object must be passed"""
if not ((isinstance(configObject, etree._ElementTree) or isinstance(
configObject, etree._Element)) and configObject is not None):
print ConfigurationError, """The configuration object input is not a valid """\
"""etree.Element object. Trying to load the object from the configuration"""\
""" file."""
self.fileLoc = fileLoc
if configObject is None:
configObject = etree.parse(fileLoc)
self.configObject = configObject
self.parallelFlag = parallelFlag
print '________________________________________________________________'
print 'PARSING CONFIG FILE'
self.configParser = ConfigParser(
self.configObject) #creates the model configuration parser
self.configParser.parse_project()
self.project = self.configParser.project
self.create_database()
print 'COMPLETED PARSING CONFIG FILE AND CREATED THE MASTER DATABASE'
print '________________________________________________________________'
try:
os.mkdir('%s%s%s' %
(self.project.location, os.path.sep, self.project.name))
except Exception, e:
print('Warning when creating folder:', e)
ppservers = ()
self.job_server = pp.Server(ppservers=ppservers, restart=True)
def resolve_consistency(self, data, seed):
print 'Data processing for use in PopGen'
popgenConfigObject = ConfigParserPopGen(self.popgenConfig)
popgenManagerObj = PopgenManager(configObject=self.popgenConfig)
popgenConfigObject.parse_project()
popgenConfigObject.parse_scenarios()
scenario = popgenConfigObject.scenarioList[0]
print 'Household vars - ', self.hhldColNames, len(self.hhldColNames)
print 'Extra household vars - ', scenario.hhldVars, len(scenario.hhldVars)
hhldVars = copy.deepcopy(self.hhldColNames)
for var in scenario.hhldVars:
if var not in hhldVars:
hhldVars.append(var)
print '\nHousehold vars - ', self.personColNames, len(self.personColNames)
print 'Extra household vars - ', scenario.personVars, len(scenario.personVars)
personVars = copy.deepcopy(self.personColNames)
for var in scenario.personVars:
if var not in personVars:
personVars.append(var)
print '\nHousehold sample file location - ', scenario.sampleUserProv.hhLocation
print 'Person sample file location - ', scenario.sampleUserProv.personLocation
# Create Index Matrix
data.sort([self.idSpec.hidName, self.idSpec.pidName])
self.create_indices(data)
self.create_col_numbers(data._colnames)
#highestHid = amax(self.hhldIndicesOfPersons[:,0])
#fHhld = csv.writer(open(scenario.sampleUserProv.hhLocation, 'w'), delimiter=",")
fHhld = open(scenario.sampleUserProv.hhLocation, 'w')
hhldVars_forPopGen = copy.deepcopy(hhldVars)
try:
hidIndex = hhldVars_forPopGen.index('houseid')
hhldVars_forPopGen[hidIndex] = 'hhid'
except ValueError, e:
print 'Error replacing "houseid" name - ', e
hidIndex = 0
def resolve_consistency(self, data, seed):
print 'Data processing for use in PopGen'
popgenConfigObject = ConfigParserPopGen(self.popgenConfig)
popgenManagerObj = PopgenManager(configObject=self.popgenConfig)
popgenConfigObject.parse_project()
popgenConfigObject.parse_scenarios()
scenario = popgenConfigObject.scenarioList[0]
print 'Household vars - ', self.hhldColNames, len(self.hhldColNames)
print 'Extra household vars - ', scenario.hhldVars, len(
scenario.hhldVars)
hhldVars = copy.deepcopy(self.hhldColNames)
for var in scenario.hhldVars:
if var not in hhldVars:
hhldVars.append(var)
print '\nHousehold vars - ', self.personColNames, len(
self.personColNames)
print 'Extra household vars - ', scenario.personVars, len(
scenario.personVars)
personVars = copy.deepcopy(self.personColNames)
for var in scenario.personVars:
if var not in personVars:
personVars.append(var)
print '\nHousehold sample file location - ', scenario.sampleUserProv.hhLocation
print 'Person sample file location - ', scenario.sampleUserProv.personLocation
# Create Index Matrix
data.sort([self.idSpec.hidName, self.idSpec.pidName])
self.create_indices(data)
self.create_col_numbers(data._colnames)
#highestHid = amax(self.hhldIndicesOfPersons[:,0])
#fHhld = csv.writer(open(scenario.sampleUserProv.hhLocation, 'w'), delimiter=",")
fHhld = open(scenario.sampleUserProv.hhLocation, 'w')
hhldVars_forPopGen = copy.deepcopy(hhldVars)
try:
hidIndex = hhldVars_forPopGen.index('houseid')
hhldVars_forPopGen[hidIndex] = 'hhid'
except ValueError, e:
print 'Error replacing "houseid" name - ', e
hidIndex = 0
def __init__(self, fileLoc=None, configObject=None, parallelFlag=1):
if fileLoc == None and configObject == None:
fileLoc = os.path.realpath('./configuration/config.xml')
print 'file location is now assigned to default - ', fileLoc
if configObject is None and fileLoc is None:
raise ConfigurationError, """The configuration input is not valid; a """\
"""location of the XML configuration file or a valid etree """\
"""object must be passed"""
if not ((isinstance(configObject, etree._ElementTree) or isinstance(
configObject, etree._Element)) and configObject is not None):
print ConfigurationError, """The configuration object input is not a valid """\
"""etree.Element object. Trying to load the object from the configuration"""\
""" file."""
self.fileLoc = fileLoc
if configObject is None:
configObject = etree.parse(fileLoc)
self.configObject = configObject
self.parallelFlag = parallelFlag
print '________________________________________________________________'
print 'PARSING CONFIG FILE'
self.configParser = ConfigParser(
self.configObject) #creates the model configuration parser
self.configParser.parse_project()
self.project = self.configParser.project
self.create_database()
print 'COMPLETED PARSING CONFIG FILE AND CREATED THE MASTER DATABASE'
print '________________________________________________________________'
try:
os.mkdir('%s%s%s' %
(self.project.location, os.path.sep, self.project.name))
except Exception, e:
print('Warning when creating folder:', e)
def resolve_consistency(self, data, seed):
print 'Data processing for use in PopGen'
popgenConfigObject = ConfigParserPopGen(self.popgenConfig)
popgenManagerObj = PopgenManager(configObject=self.popgenConfig)
popgenConfigObject.parse_project()
popgenConfigObject.parse_scenarios()
scenario = popgenConfigObject.scenarioList[0]
print 'Household vars - ', self.hhldColNames, len(self.hhldColNames)
print 'Extra household vars - ', scenario.hhldVars, len(scenario.hhldVars)
hhldVars = copy.deepcopy(self.hhldColNames)
for var in scenario.hhldVars:
if var not in hhldVars:
hhldVars.append(var)
print '\nHousehold vars - ', self.personColNames, len(self.personColNames)
print 'Extra household vars - ', scenario.personVars, len(scenario.personVars)
personVars = copy.deepcopy(self.personColNames)
for var in scenario.personVars:
if var not in personVars:
personVars.append(var)
print '\nHousehold sample file location - ', scenario.sampleUserProv.hhLocation
print 'Person sample file location - ', scenario.sampleUserProv.personLocation
# Create Index Matrix
data.sort([self.idSpec.hidName, self.idSpec.pidName])
self.create_indices(data)
self.create_col_numbers(data._colnames)
highestHid = amax(self.hhldIndicesOfPersons[:,0])
#fHhld = csv.writer(open(scenario.sampleUserProv.hhLocation, 'w'), delimiter=",")
fHhld = open(scenario.sampleUserProv.hhLocation, 'w')
hhldVars_forPopGen = copy.deepcopy(hhldVars)
hidIndex = hhldVars_forPopGen.index('houseid')
hhldVars_forPopGen[hidIndex] = 'hhid'
hhldVars_forPopGen.insert(hidIndex+1,'serialno')
hhldVars_forPopGen.insert(0,'pumano')
hhldVars_forPopGen.insert(0,'state')
print 'new hhldvars', hhldVars_forPopGen
#fHhld.writerow(hhldVars_forPopGen)
hhldVars_str = ''
for var in hhldVars_forPopGen:
hhldVars_str += '%s,' %var
hhldVars_str = hhldVars_str[:-1]
#fHhld.writerow(hhldVars_forPopGen)
fHhld.write(hhldVars_str)
fHhld.write('\n')
personVars_forPopGen = copy.deepcopy(personVars)
#fPerson = csv.writer(open(scenario.sampleUserProv.personLocation, 'w'), delimiter=",")
fPerson = open(scenario.sampleUserProv.personLocation, 'w')
hidIndex = personVars_forPopGen.index('houseid')
personVars_forPopGen[hidIndex] = 'hhid'
pidIndex = personVars_forPopGen.index('personid')
personVars_forPopGen[pidIndex] = 'pnum'
personVars_forPopGen.insert(hidIndex+1,'serialno')
personVars_forPopGen.insert(0,'pumano')
personVars_forPopGen.insert(0,'state')
print 'new personvars', personVars_forPopGen
personVars_str = ''
for var in personVars_forPopGen:
personVars_str += '%s,' %var
personVars_str = personVars_str[:-1]
#fPerson.writerow(personVars_forPopGen)
fPerson.write(personVars_str)
fPerson.write('\n')
hhldCols = data.columns(hhldVars)
personCols = data.columns(personVars)
ti = time.time()
hhldRow_forPopGen = [0] * len(hhldVars_forPopGen)
hhldRow_forPopGen[0] = 24
hhldRow_forPopGen[1] = 99
personRow_forPopGen = [0] * len(personVars_forPopGen)
personRow_forPopGen[0] = 24
personRow_forPopGen[1] = 99
for hhldIndex in self.hhldIndicesOfPersons:
firstPersonRec = hhldIndex[1]
lastPersonRec = hhldIndex[2]
#personRecs = DataArray(data.data[firstPersonRec:lastPersonRec, :], data.varnames)
hhldRow = hhldCols.data[firstPersonRec].astype(int)
personRows = personCols.data[firstPersonRec:lastPersonRec].astype(int)
#print 'Hhlds\n', hhldRow
#print 'Persons\n', personRows
# Writing out rows for the household sample file
hhldRow_forPopGen[3:] = list(hhldRow)
hhldRow_forPopGen[2] = hhldRow_forPopGen[3]
#fHhld.writerow(list(hhldRow_forPopGen))
fHhld.write(','.join(map(str, list(hhldRow_forPopGen))))
fHhld.write('\n')
# Writing out rows for the person sample file
for personRow in personRows:
personRow_forPopGen[3:] = personRow
personRow_forPopGen[2] = personRow_forPopGen[3]
#fPerson.writerow(list(personRow_forPopGen))
fPerson.write(','.join(map(str, list(personRow_forPopGen))))
fPerson.write('\n')
fHhld.close()
fPerson.close()
print '\t Sample input files created in - %.4f' %(time.time()-ti)
print 'Startint to synthesize population - '
ti = time.time()
popgenManagerObj = PopgenManager(configObject=self.popgenConfig)
popgenManagerObj.run_scenarios()
print '\t Synthesis complete in - %.4f' %(time.time()-ti)
hhldSynLoc = scenario.synHousingTableNameLoc.location
hhldSynFileName = scenario.synHousingTableNameLoc.name
fHhldSyn = csv.reader(open('%s%s%s.dat' %(hhldSynLoc, os.path.sep, hhldSynFileName), 'r'), delimiter='\t')
hhldSynArr = self.load_file(fHhldSyn)
print hhldSynArr[:5,:]
hhldSynVars = self.parse_meta_file('%s%s%s_meta.txt' %(hhldSynLoc, os.path.sep, hhldSynFileName))
print 'Old var names - ', hhldSynVars
hidIndex_popgen = hhldSynVars.index('hhid')
hhldSynVars[hidIndex_popgen] = 'houseid'
print 'New var names - ', hhldSynVars
persSynLoc = scenario.synPersTableNameLoc.location
persSynFileName = scenario.synPersTableNameLoc.name
fPersSyn = csv.reader(open('%s%s%s.dat' %(persSynLoc, os.path.sep, persSynFileName), 'r'), delimiter='\t')
persSynArr = self.load_file(fPersSyn)
print persSynArr[:5,:]
personSynVars = self.parse_meta_file('%s%s%s_meta.txt' %(persSynLoc, os.path.sep, persSynFileName))
print 'Old var names - ', personSynVars
hidIndex_popgen = personSynVars.index('hhid')
personSynVars[hidIndex_popgen] = 'houseid'
pidIndex_popgen = personSynVars.index('pnum')
personSynVars[pidIndex_popgen] = 'personid'
print 'New var names - ', personSynVars
print '\nHousehold sample file location - ', scenario.synHousingTableNameLoc
print 'Person sample file location - ', scenario.synPersTableNameLoc
return DataArray(persSynArr, personSynVars), DataArray(hhldSynArr, hhldSynVars)
def resolve_consistency(self, data, seed):
print 'Data processing for use in PopGen'
popgenConfigObject = ConfigParserPopGen(self.popgenConfig)
popgenManagerObj = PopgenManager(configObject=self.popgenConfig)
popgenConfigObject.parse_project()
popgenConfigObject.parse_scenarios()
scenario = popgenConfigObject.scenarioList[0]
print 'Household vars - ', self.hhldColNames, len(self.hhldColNames)
print 'Extra household vars - ', scenario.hhldVars, len(
scenario.hhldVars)
hhldVars = copy.deepcopy(self.hhldColNames)
for var in scenario.hhldVars:
if var not in hhldVars:
hhldVars.append(var)
print '\nHousehold vars - ', self.personColNames, len(
self.personColNames)
print 'Extra household vars - ', scenario.personVars, len(
scenario.personVars)
personVars = copy.deepcopy(self.personColNames)
for var in scenario.personVars:
if var not in personVars:
personVars.append(var)
print '\nHousehold sample file location - ', scenario.sampleUserProv.hhLocation
print 'Person sample file location - ', scenario.sampleUserProv.personLocation
# Create Index Matrix
data.sort([self.idSpec.hidName, self.idSpec.pidName])
self.create_indices(data)
self.create_col_numbers(data._colnames)
highestHid = amax(self.hhldIndicesOfPersons[:, 0])
#fHhld = csv.writer(open(scenario.sampleUserProv.hhLocation, 'w'), delimiter=",")
fHhld = open(scenario.sampleUserProv.hhLocation, 'w')
hhldVars_forPopGen = copy.deepcopy(hhldVars)
hidIndex = hhldVars_forPopGen.index('houseid')
hhldVars_forPopGen[hidIndex] = 'hhid'
hhldVars_forPopGen.insert(hidIndex + 1, 'serialno')
hhldVars_forPopGen.insert(0, 'pumano')
hhldVars_forPopGen.insert(0, 'state')
print 'new hhldvars', hhldVars_forPopGen
#fHhld.writerow(hhldVars_forPopGen)
hhldVars_str = ''
for var in hhldVars_forPopGen:
hhldVars_str += '%s,' % var
hhldVars_str = hhldVars_str[:-1]
#fHhld.writerow(hhldVars_forPopGen)
fHhld.write(hhldVars_str)
fHhld.write('\n')
personVars_forPopGen = copy.deepcopy(personVars)
#fPerson = csv.writer(open(scenario.sampleUserProv.personLocation, 'w'), delimiter=",")
fPerson = open(scenario.sampleUserProv.personLocation, 'w')
hidIndex = personVars_forPopGen.index('houseid')
personVars_forPopGen[hidIndex] = 'hhid'
pidIndex = personVars_forPopGen.index('personid')
personVars_forPopGen[pidIndex] = 'pnum'
personVars_forPopGen.insert(hidIndex + 1, 'serialno')
personVars_forPopGen.insert(0, 'pumano')
personVars_forPopGen.insert(0, 'state')
print 'new personvars', personVars_forPopGen
personVars_str = ''
for var in personVars_forPopGen:
personVars_str += '%s,' % var
personVars_str = personVars_str[:-1]
#fPerson.writerow(personVars_forPopGen)
fPerson.write(personVars_str)
fPerson.write('\n')
hhldCols = data.columns(hhldVars)
personCols = data.columns(personVars)
ti = time.time()
hhldRow_forPopGen = [0] * len(hhldVars_forPopGen)
hhldRow_forPopGen[0] = 24
hhldRow_forPopGen[1] = 99
personRow_forPopGen = [0] * len(personVars_forPopGen)
personRow_forPopGen[0] = 24
personRow_forPopGen[1] = 99
for hhldIndex in self.hhldIndicesOfPersons:
firstPersonRec = hhldIndex[1]
lastPersonRec = hhldIndex[2]
#personRecs = DataArray(data.data[firstPersonRec:lastPersonRec, :], data.varnames)
hhldRow = hhldCols.data[firstPersonRec].astype(int)
personRows = personCols.data[firstPersonRec:lastPersonRec].astype(
int)
#print 'Hhlds\n', hhldRow
#print 'Persons\n', personRows
# Writing out rows for the household sample file
hhldRow_forPopGen[3:] = list(hhldRow)
hhldRow_forPopGen[2] = hhldRow_forPopGen[3]
#fHhld.writerow(list(hhldRow_forPopGen))
fHhld.write(','.join(map(str, list(hhldRow_forPopGen))))
fHhld.write('\n')
# Writing out rows for the person sample file
for personRow in personRows:
personRow_forPopGen[3:] = personRow
personRow_forPopGen[2] = personRow_forPopGen[3]
#fPerson.writerow(list(personRow_forPopGen))
fPerson.write(','.join(map(str, list(personRow_forPopGen))))
fPerson.write('\n')
fHhld.close()
fPerson.close()
print '\t Sample input files created in - %.4f' % (time.time() - ti)
print 'Startint to synthesize population - '
ti = time.time()
popgenManagerObj = PopgenManager(configObject=self.popgenConfig)
popgenManagerObj.run_scenarios()
print '\t Synthesis complete in - %.4f' % (time.time() - ti)
hhldSynLoc = scenario.synHousingTableNameLoc.location
hhldSynFileName = scenario.synHousingTableNameLoc.name
fHhldSyn = csv.reader(open(
'%s%s%s.dat' % (hhldSynLoc, os.path.sep, hhldSynFileName), 'r'),
delimiter='\t')
hhldSynArr = self.load_file(fHhldSyn)
print hhldSynArr[:5, :]
hhldSynVars = self.parse_meta_file(
'%s%s%s_meta.txt' % (hhldSynLoc, os.path.sep, hhldSynFileName))
print 'Old var names - ', hhldSynVars
hidIndex_popgen = hhldSynVars.index('hhid')
hhldSynVars[hidIndex_popgen] = 'houseid'
print 'New var names - ', hhldSynVars
persSynLoc = scenario.synPersTableNameLoc.location
persSynFileName = scenario.synPersTableNameLoc.name
fPersSyn = csv.reader(open(
'%s%s%s.dat' % (persSynLoc, os.path.sep, persSynFileName), 'r'),
delimiter='\t')
persSynArr = self.load_file(fPersSyn)
print persSynArr[:5, :]
personSynVars = self.parse_meta_file(
'%s%s%s_meta.txt' % (persSynLoc, os.path.sep, persSynFileName))
print 'Old var names - ', personSynVars
hidIndex_popgen = personSynVars.index('hhid')
personSynVars[hidIndex_popgen] = 'houseid'
pidIndex_popgen = personSynVars.index('pnum')
personSynVars[pidIndex_popgen] = 'personid'
print 'New var names - ', personSynVars
print '\nHousehold sample file location - ', scenario.synHousingTableNameLoc
print 'Person sample file location - ', scenario.synPersTableNameLoc
return DataArray(persSynArr,
personSynVars), DataArray(hhldSynArr, hhldSynVars)
