def run_estimation(self, estimation_config, model_name, save_estimation_results=True):
config = Baseline()
config.merge(estimation_config)
config['config_changes_for_estimation'] = ConfigChangesForEstimation()
logger.start_block('Estimating %s' % model_name)
try:
estimator = UrbansimEstimationRunner(
models[model_name][0],
specification_module=models[model_name][1], model_group=models[model_name][2],
configuration=config,
save_estimation_results=save_estimation_results
)
estimator.estimate()
finally:
logger.end_block()
def test_estimation(self):
for model_name in [
'real_estate_price_model', 'household_location_choice_model',
'business_location_choice_model',
'building_location_choice_model'
]:
if type(model_name) == tuple:
model_name, group_member = model_name
else:
group_member = None
estimator = EstimationRunner(model=model_name,
model_group=group_member,
xml_configuration=self.xml_config,
configuration=None,
save_estimation_results=True)
estimator.estimate()
def test_estimation(self):
for model_name in ['real_estate_price_model',
'household_location_choice_model',
'business_location_choice_model',
'building_location_choice_model'
]:
if type(model_name)==tuple:
model_name, group_member = model_name
else:
group_member = None
estimator = EstimationRunner(model=model_name,
model_group=group_member,
xml_configuration=self.xml_config,
configuration = None,
save_estimation_results=True
)
estimator.estimate()
def run_estimation(self,
estimation_config,
model_name,
save_estimation_results=True):
config = Baseline()
config.merge(estimation_config)
config['config_changes_for_estimation'] = ConfigChangesForEstimation()
logger.start_block('Estimating %s' % model_name)
try:
estimator = UrbansimEstimationRunner(
models[model_name][0],
specification_module=models[model_name][1],
model_group=models[model_name][2],
configuration=config,
save_estimation_results=save_estimation_results)
estimator.estimate()
finally:
logger.end_block()
def test_estimation(self):
for model_name in ['real_estate_price_model',
'household_location_choice_model',
('employment_location_choice_model', 'home_based'),
('employment_location_choice_model', 'non_home_based'),
'residential_development_project_location_choice_model',
'non_residential_development_project_location_choice_model',
]:
if type(model_name)==tuple:
model_name, group_member = model_name
else:
group_member = None
estimator = EstimationRunner(model=model_name,
model_group=group_member,
xml_configuration=self.xml_config,
configuration = None
)
estimator.estimate()
def test_estimation(self):
for model_name in [
'real_estate_price_model',
'household_location_choice_model',
('employment_location_choice_model', 'home_based'),
('employment_location_choice_model', 'non_home_based'),
'residential_development_project_location_choice_model',
'non_residential_development_project_location_choice_model',
]:
if type(model_name) == tuple:
model_name, group_member = model_name
else:
group_member = None
estimator = EstimationRunner(model=model_name,
model_group=group_member,
xml_configuration=self.xml_config,
configuration=None)
estimator.estimate()
def run(self):
# Start the estimation. This code adapted from urbansim/tools/start_estimation.py
# statusdir is a temporary directory into which to write a status file
# regarding the progress of the simulation - the progress bar reads this file
statusfile = None
succeeded = False
try:
# get the configuration for estimations
estimation_section = self.xml_config.get_section('model_manager')
estimation_config = estimation_section['estimation_config']
self.config = estimation_config
# TODO: put this option into post run dialog
save_results = estimation_config['save_estimation_results']
# If we've paused the estimation, wait 10 seconds, and see if we are unpaused. If we've cancelled,
# exit the loop. Note that this is a fairly coarse level of pause/resume/stop (at the level of
# estimating an entire model, rather than a bit of a model).
while self.paused and not self.cancelled:
time.sleep(10)
if not self.cancelled:
self.er = EstimationRunner(
model=self.model_name,
specification_module=None,
xml_configuration=self.xml_config,
model_group=self.model_group,
configuration=None,
save_estimation_results=save_results)
self.running = True
self.er.estimate()
self.running = False
succeeded = True
except:
succeeded = False
self.running = False
errorInfo = formatExceptionInfo(
custom_message='Unexpected Error From Estimation')
self.errorCallback(errorInfo)
if statusfile is not None:
os.remove(statusfile)
del self.er
gc.collect()
self.finishedCallback(succeeded)
def test_estimation(self):
for model_name in [
"residential_building_type_choice_model",
"tenure_choice_model",
"work_at_home_choice_model",
"workplace_choice_model_for_resident",
"auto_ownership_choice_model",
"hbw_mode_choice_model",
"real_estate_price_model",
"household_location_choice_model",
"shopping_destination_choice_model",
("employment_location_choice_model", "home_based"),
("employment_location_choice_model", "non_home_based"),
]:
if type(model_name) == tuple:
model_name, group_member = model_name
else:
group_member = None
estimator = EstimationRunner(
model=model_name, model_group=group_member, xml_configuration=self.xml_config, configuration=None
)
estimator.estimate()
def test_estimation(self):
for model_name in ['residential_building_type_choice_model',
'tenure_choice_model',
'work_at_home_choice_model',
'workplace_choice_model_for_resident',
'auto_ownership_choice_model',
'hbw_mode_choice_model',
'real_estate_price_model',
'household_location_choice_model',
'shopping_destination_choice_model',
('employment_location_choice_model', 'home_based'),
('employment_location_choice_model', 'non_home_based')
]:
if type(model_name)==tuple:
model_name, group_member = model_name
else:
group_member = None
estimator = EstimationRunner(model=model_name,
model_group=group_member,
xml_configuration=self.xml_config,
configuration = None
)
estimator.estimate()
def test_estimation(self):
# open the configuration for seattle_parcel.xml
seattle_parcel_dir = __import__('seattle_parcel').__path__[0]
xml_config = XMLConfiguration(
os.path.join(seattle_parcel_dir, 'configs', 'seattle_parcel.xml'))
for model_name in [
'real_estate_price_model', 'household_location_choice_model'
]:
er = EstimationRunner(model=model_name,
xml_configuration=xml_config,
configuration=None)
er.estimate()
# test run with group members
er = EstimationRunner(model='employment_location_choice_model',
xml_configuration=xml_config,
model_group='home_based',
configuration=None)
er.estimate()
def test_estimation(self):
# open the configuration for seattle_parcel.xml
seattle_parcel_dir = __import__('seattle_parcel').__path__[0]
xml_config = XMLConfiguration(os.path.join(seattle_parcel_dir, 'configs', 'seattle_parcel.xml'))
for model_name in ['real_estate_price_model',
'household_location_choice_model']:
er = EstimationRunner(model=model_name, xml_configuration=xml_config, configuration=None)
er.estimate()
# test run with group members
er = EstimationRunner(model='employment_location_choice_model',
xml_configuration=xml_config,
model_group = 'home_based',
configuration=None)
er.estimate()
def run(self):
# Start the estimation. This code adapted from urbansim/tools/start_estimation.py
# statusdir is a temporary directory into which to write a status file
# regarding the progress of the simulation - the progress bar reads this file
statusfile = None
succeeded = False
try:
# get the configuration for estimations
estimation_section = self.xml_config.get_section('model_manager')
estimation_config = estimation_section['estimation_config']
self.config = estimation_config
# TODO: put this option into post run dialog
save_results = estimation_config['save_estimation_results']
# If we've paused the estimation, wait 10 seconds, and see if we are unpaused. If we've cancelled,
# exit the loop. Note that this is a fairly coarse level of pause/resume/stop (at the level of
# estimating an entire model, rather than a bit of a model).
while self.paused and not self.cancelled:
time.sleep(10)
if not self.cancelled:
self.er = EstimationRunner(model = self.model_name,
specification_module = None,
xml_configuration = self.xml_config,
model_group = self.model_group,
configuration = None,
save_estimation_results=save_results)
self.running = True
self.er.estimate()
self.running = False
succeeded = True
except:
succeeded = False
self.running = False
errorInfo = formatExceptionInfo(custom_message = 'Unexpected Error From Estimation')
self.errorCallback(errorInfo)
if statusfile is not None:
os.remove(statusfile)
del self.er
gc.collect()
self.finishedCallback(succeeded)
'repm': [
'real_estate_price_model',
'sanfrancisco.estimation.repm_specification', None
],
"BLDGLCM-nonres": [
"building_location_choice_model",
'sanfrancisco.estimation.bldglcm_specification', "nonresidential"
],
"BLDGLCM-res": [
"building_location_choice_model",
'sanfrancisco.estimation.bldglcm_specification', "residential"
],
}
if __name__ == '__main__':
# model = 'repm'
# model = 'hlcm'
# model = 'blcm'
# model = "BLDGLCM-nonres"
model = "BLDGLCM-res"
config = Baseline()
config.merge(my_configuration)
config['config_changes_for_estimation'] = ConfigChangesForEstimation()
er = EstimationRunner(models[model][0],
specification_module=models[model][1],
model_group=models[model][2],
configuration=config,
save_estimation_results=True)
er.estimate()
if options.model_name is None:
raise StandardError, "Model name (argument -m) must be given."
if (options.configuration_path is None) and (options.xml_configuration is None):
raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given."
if (options.specification is None) and (options.xml_configuration is None):
logger.log_warning("No specification given (arguments -s or -x). Specification taken from the cache.")
if options.xml_configuration is not None:
xconfig = XMLConfiguration(options.xml_configuration)
else:
xconfig = None
if options.configuration_path is None:
config = None
else:
config = get_config_from_opus_path(options.configuration_path)
estimator = EstimationRunner(model=options.model_name,
specification_module=options.specification,
xml_configuration=xconfig,
model_group=options.model_group,
configuration=config,
save_estimation_results=options.save_results)
estimator.estimate()
er = estimator
ds = er.get_data_as_dataset()
ds.summary()
ds.get_attribute("ln_residential_units").shape
estimator.create_prediction_success_table(summarize_by="fazdistrict_id=building.disaggregate(faz.fazdistrict_id, intermediates=[zone, parcel])")
estimator.create_prediction_success_table(summarize_by="area_type_id=building.disaggregate(zone.area_type_id, intermediates=[parcel])")
# Make you private changes in sanfrancisco.configs.baseline_estimation.
#!!!!!!!!!!!!
from sanfrancisco.estimation.my_estimation_config import my_configuration
from urbansim.estimation.estimation_runner import EstimationRunner
from sanfrancisco.configs.baseline import Baseline
from sanfrancisco.configs.config_changes_for_estimation import ConfigChangesForEstimation
models = {
'hlcm': ['household_location_choice_model', 'sanfrancisco.estimation.hlcm_specification', None],
'blcm': ['business_location_choice_model', 'sanfrancisco.estimation.blcm_specification', None],
'repm': ['real_estate_price_model', 'sanfrancisco.estimation.repm_specification', None],
"BLDGLCM-nonres": ["building_location_choice_model", 'sanfrancisco.estimation.bldglcm_specification', "nonresidential"],
"BLDGLCM-res": ["building_location_choice_model", 'sanfrancisco.estimation.bldglcm_specification', "residential"],
}
if __name__ == '__main__':
# model = 'repm'
# model = 'hlcm'
# model = 'blcm'
# model = "BLDGLCM-nonres"
model = "BLDGLCM-res"
config = Baseline()
config.merge(my_configuration)
config['config_changes_for_estimation'] = ConfigChangesForEstimation()
er = EstimationRunner(models[model][0], specification_module=models[model][1], model_group=models[model][2],
configuration=config, save_estimation_results=True)
er.estimate()
class OpusEstimation(object):
# xml_config, model_name, model_group
def __init__(self, xml_config, model_name, model_group=None):
'''
@param xml_config (XMLConfiguration): xml configuration object for the model
@param model_name (String): the name of the model to run
@param model_group (String): name of the sub model group to run
'''
dummy_callback = lambda x: None
self.xml_config = xml_config
self.model_name = model_name
self.model_group = model_group
self.er = None
self.progressCallback = dummy_callback
self.finishedCallback = dummy_callback
self.errorCallback = dummy_callback
self.guiElement = None
self.config = None
self.statusfile = None
self.firstRead = True
self.running = False
self.paused = False
self.cancelled = False
self.statusfile = None
def pause(self):
self.paused = True
print "Pause pressed"
# Can access the estimation manager via self.er
def resume(self):
self.paused = False
print "Resume pressed"
# Can access the estimation manager via self.er
def cancel(self):
self.running = False
self.paused = False
self.cancelled = True
print "Cancel pressed"
# Can access the estimation manager via self.er
def run(self):
# Start the estimation. This code adapted from urbansim/tools/start_estimation.py
# statusdir is a temporary directory into which to write a status file
# regarding the progress of the simulation - the progress bar reads this file
statusfile = None
succeeded = False
try:
# get the configuration for estimations
estimation_section = self.xml_config.get_section('model_manager')
estimation_config = estimation_section['estimation_config']
self.config = estimation_config
# TODO: put this option into post run dialog
save_results = estimation_config['save_estimation_results']
# If we've paused the estimation, wait 10 seconds, and see if we are unpaused. If we've cancelled,
# exit the loop. Note that this is a fairly coarse level of pause/resume/stop (at the level of
# estimating an entire model, rather than a bit of a model).
while self.paused and not self.cancelled:
time.sleep(10)
if not self.cancelled:
self.er = EstimationRunner(
model=self.model_name,
specification_module=None,
xml_configuration=self.xml_config,
model_group=self.model_group,
configuration=None,
save_estimation_results=save_results)
self.running = True
self.er.estimate()
self.running = False
succeeded = True
except:
succeeded = False
self.running = False
errorInfo = formatExceptionInfo(
custom_message='Unexpected Error From Estimation')
self.errorCallback(errorInfo)
if statusfile is not None:
os.remove(statusfile)
del self.er
gc.collect()
self.finishedCallback(succeeded)
def _compute_progress(self):
if self.statusfile is None:
return {"percentage": 0, "message": "Estimation initializing..."}
#TODO: Compute percent progress for the progress bar.
percentage = 0
message = 'Estimation initializing...'
return {"percentage": percentage, "message": message}
def _get_current_log(self, key):
newKey = key
# We attempt to keep up on the current progress of the model run. We pass into this
# function an initial "key" value of 0 and expect to get back a new "key" after the
# function returns. It is up to us in this function to use this key to determine
# what has happened since last time this function was called.
# In this example we use the key to indicate where in a logfile we last stopped reading
# and seek into that file point and read to the end of the file and append to the
# log text edit field in the GUI.
if self.config is not None and 'cache_directory' in self.config:
try:
base_year = self.config['base_year']
f = open(
os.path.join(self.config['cache_directory'],
'year_%d_log.txt' % base_year))
f.seek(key)
lines = f.read()
newKey = f.tell()
if newKey != key:
self.guiElement.logText.append(lines)
f.close()
except IOError:
if self.firstRead == True:
self.guiElement.logText.append("No logfile yet")
self.firstRead = False
else:
self.guiElement.logText.insertPlainText(QString("."))
#self.guiElement.logText.append("ping")
return newKey
#numpy.setmember1d(household.tenure, (1,3)) * \ ## buggy
"(( (psrc_parcel.household.building_type_id==4) + (psrc_parcel.household.building_type_id==11) + (psrc_parcel.household.building_type_id==12) + (psrc_parcel.household.building_type_id==19)) >= 1) * " + \
#numpy.setmember1d(psrc_parcel.household.building_type_id, (4, 11, 12, 19)) * \ ##ideally the above line can be replaced with this, but this is buggy
"(household.aggregate((person.worker1==1)*(urbansim_parcel.person.is_non_home_based_worker_with_job)))",
"customized_run_filter= household.aggregate(urbansim_parcel.person.is_non_home_based_worker)>=1"
]
xmlconfig = XMLConfiguration(options.xml_configuration)
## training data (start_estimation)
training_data = []
for h, hierarchy in enumerate(options.meta_models):
model_data = []
for i, model_name in enumerate(hierarchy):
logger.start_block('%s' % model_name)
estimator = EstimationRunner(model=model_name,
specification_module=None,
xml_configuration=xmlconfig,
model_group=None,
configuration=None,
save_estimation_results=True)
#estimator = EstimationRunner(model=model_name,
#xml_configuration=xmlconfig,
#save_estimation_results=False)
estimator.estimate()
m = estimator.get_model()
model_system = estimator.model_system
dataset_pool=model_system.run_year_namespace["dataset_pool"]
submodels = m.model_interaction.get_submodels()
assert len(submodels) == 1
submodel = submodels[0]
import wingdbstub
except:
pass
option_group = EstimationOptionGroup()
parser = option_group.parser
(options, args) = parser.parse_args()
if options.model_name is None:
raise StandardError, "Model name (argument -m) must be given."
if (options.configuration_path is None) and (options.xml_configuration is
None):
raise StandardError, "Configuration path (argument -c) or XML configuration (argument -x) must be given."
if (options.specification is None) and (options.xml_configuration is None):
logger.log_warning(
"No specification given (arguments -s or -x). Specification taken from the cache."
)
if options.xml_configuration is not None:
xconfig = XMLConfiguration(options.xml_configuration)
else:
xconfig = None
if options.configuration_path is None:
config = None
else:
config = get_config_from_opus_path(options.configuration_path)
estimator = EstimationRunner(model=options.model_name,
specification_module=options.specification,
xml_configuration=xconfig,
model_group=options.model_group,
configuration=config,
save_estimation_results=options.save_results)
estimator.estimate()
er = estimator
'hlcm': ['household_location_choice_model', 'psrc.estimation.psrc.estimation_HLCM_variables', None],
'elcm-industrial': ['employment_location_choice_model', 'psrc.estimation.psrc.estimation_ELCM_variables', 'industrial'],
'elcm-commercial': ['employment_location_choice_model', 'psrc.estimation.psrc.estimation_ELCM_variables', 'commercial'],
'elcm-home_based': ['employment_location_choice_model', 'psrc.estimation.psrc.estimation_ELCM_variables', 'home_based'],
'lpm': ['land_price_model', 'psrc.estimation.psrc.estimation_LPM_variables', None],
'dplcm-industrial': ['development_project_location_choice_model', 'psrc.estimation.psrc.estimation_DPLCM_variables', 'industrial'],
'dplcm-commercial': ['development_project_location_choice_model', 'psrc.estimation.psrc.estimation_DPLCM_variables', 'commercial'],
'dplcm-residential': ['development_project_location_choice_model', 'psrc.estimation.psrc.estimation_DPLCM_variables', 'residential'],
'rlsm': ['residential_land_share_model', 'psrc.estimation.psrc.estimation_RLSM_variables', None],
}
if __name__ == '__main__':
#model = 'hlcm'
#model = 'elcm-industrial'
#model = 'elcm-commercial'
#model = 'elcm-home_based'
model = 'dplcm-industrial'
#model = 'dplcm-commercial'
#model ='dplcm-residential'
#model = 'lpm'
#model = 'rlsm'
config = Baseline()
config.merge(my_configuration)
config['config_changes_for_estimation'] = ConfigChangesForEstimation()
estimator = EstimationRunner(models[model][0], specification_module=models[model][1], model_group=models[model][2],
configuration=config,
save_estimation_results=True)
estimator.estimate()
#estimator.reestimate()
#estimator.reestimate(submodels=[])
class OpusEstimation(object):
# xml_config, model_name, model_group
def __init__(self, xml_config, model_name, model_group = None):
'''
@param xml_config (XMLConfiguration): xml configuration object for the model
@param model_name (String): the name of the model to run
@param model_group (String): name of the sub model group to run
'''
dummy_callback = lambda x: None
self.xml_config = xml_config
self.model_name = model_name
self.model_group = model_group
self.er = None
self.progressCallback = dummy_callback
self.finishedCallback = dummy_callback
self.errorCallback = dummy_callback
self.guiElement = None
self.config = None
self.statusfile = None
self.firstRead = True
self.running = False
self.paused = False
self.cancelled = False
self.statusfile = None
def pause(self):
self.paused = True
print "Pause pressed"
# Can access the estimation manager via self.er
def resume(self):
self.paused = False
print "Resume pressed"
# Can access the estimation manager via self.er
def cancel(self):
self.running = False
self.paused = False
self.cancelled = True
print "Cancel pressed"
# Can access the estimation manager via self.er
def run(self):
# Start the estimation. This code adapted from urbansim/tools/start_estimation.py
# statusdir is a temporary directory into which to write a status file
# regarding the progress of the simulation - the progress bar reads this file
statusfile = None
succeeded = False
try:
# get the configuration for estimations
estimation_section = self.xml_config.get_section('model_manager')
estimation_config = estimation_section['estimation_config']
self.config = estimation_config
# TODO: put this option into post run dialog
save_results = estimation_config['save_estimation_results']
# If we've paused the estimation, wait 10 seconds, and see if we are unpaused. If we've cancelled,
# exit the loop. Note that this is a fairly coarse level of pause/resume/stop (at the level of
# estimating an entire model, rather than a bit of a model).
while self.paused and not self.cancelled:
time.sleep(10)
if not self.cancelled:
self.er = EstimationRunner(model = self.model_name,
specification_module = None,
xml_configuration = self.xml_config,
model_group = self.model_group,
configuration = None,
save_estimation_results=save_results)
self.running = True
self.er.estimate()
self.running = False
succeeded = True
except:
succeeded = False
self.running = False
errorInfo = formatExceptionInfo(custom_message = 'Unexpected Error From Estimation')
self.errorCallback(errorInfo)
if statusfile is not None:
os.remove(statusfile)
del self.er
gc.collect()
self.finishedCallback(succeeded)
def _compute_progress(self):
if self.statusfile is None:
return {"percentage":0,"message":"Estimation initializing..."}
#TODO: Compute percent progress for the progress bar.
percentage = 0
message = 'Estimation initializing...'
return {"percentage":percentage,"message":message}
def _get_current_log(self, key):
newKey = key
# We attempt to keep up on the current progress of the model run. We pass into this
# function an initial "key" value of 0 and expect to get back a new "key" after the
# function returns. It is up to us in this function to use this key to determine
# what has happened since last time this function was called.
# In this example we use the key to indicate where in a logfile we last stopped reading
# and seek into that file point and read to the end of the file and append to the
# log text edit field in the GUI.
if self.config is not None and 'cache_directory' in self.config:
try:
base_year = self.config['base_year']
f = open(os.path.join(self.config['cache_directory'],'year_%d_log.txt' % base_year))
f.seek(key)
lines = f.read()
newKey = f.tell()
if newKey != key:
self.guiElement.logText.append(lines)
f.close()
except IOError:
if self.firstRead == True:
self.guiElement.logText.append("No logfile yet")
self.firstRead = False
else:
self.guiElement.logText.insertPlainText(QString("."))
#self.guiElement.logText.append("ping")
return newKey