def plot_coefficients(self, submodel=-2, exclude_constant=True, eqidx=0, plot=True,
store_values_to_file=None):
""" Plot a barchart of coefficient values. This can be used in a regression model,
when coefficients are standardized
(i.e. using the estimation module opus_core.estimate_linear_regression_standardized).
store_values_to_file can be a file name where the values are stored.
"""
coef = self.get_coefficients(submodel)
values = coef.get_coefficient_values()
names = coef.get_coefficient_names()
sd = coef.get_standard_errors()
idx=ones(names.shape[1], dtype="bool")
if exclude_constant:
pos = coef.get_constants_positions()
if pos.size > 0:
idx[pos]=0
if store_values_to_file is not None:
n = idx.sum()
result = concatenate((reshape(names[eqidx, idx], (n,1)),
reshape(values[eqidx, idx], (n,1)),
reshape(sd[eqidx, idx], (n,1))), axis=1)
write_to_text_file(store_values_to_file, array(['coefficient_name', 'estimate', 'standard_error']),
delimiter='\t')
write_table_to_text_file(store_values_to_file, result, delimiter='\t', mode='a')
if plot:
plot_barchart(values[eqidx, idx], labels = names[eqidx, idx], errors=sd[eqidx, idx])
else:
return {'names': names[eqidx, idx], 'values': values[eqidx, idx], 'errors': sd[eqidx, idx]}
def estimate_variance(self):
mode="wb"
for l in self.mu.keys():
self.v[l] = zeros(self.number_of_runs, dtype=float32)
for i in range(self.number_of_runs):
self.v[l][i] = ((self.y[l] - self.ahat[l] - self.mu[l][:,i])**2.0).mean()
if l > 0:
mode="ab" # add to existing file
write_to_text_file(os.path.join(self.output_directory, self.variance_file_name),
self.v[l], mode=mode, delimiter=" ")
def estimate_bias(self):
mode="wb"
for l in self.mu.keys():
self.ahat[l] = (reshape(self.y[l], (self.y[l].size,1)) - self.mu[l]).mean()
# for zone-specific bias
#self.ahat[l] = mean(reshape(self.y[l], (self.y[l].size,1)) - self.mu[l], axis=1)
if l > 0:
mode="ab" # add to existing file
write_to_text_file(os.path.join(self.output_directory, self.bias_file_name),
array([self.ahat[l]]), mode=mode)
def create_file_cache_directories(directory, prefix='', file_name='cache_directories'):
logger.start_block('Creating file %s in %s' % (file_name, directory))
all_dirs = os.listdir(directory)
all_dirs = [x for x in all_dirs if x.startswith(prefix)]
if not prefix.startswith('.'):
all_dirs = [x for x in all_dirs if not x.startswith('.')]
for i in range(len(all_dirs)):
all_dirs[i] = os.path.join(directory, all_dirs[i])
write_to_text_file(os.path.join(directory, file_name), all_dirs)
logger.end_block()
def export_bm_parameters_of_one_quantity(self, quantity_of_interest, filepath, add_to_file=True, run_index=None):
variable_list = self.get_variable_names()
quantity_index = variable_list.index(quantity_of_interest)
mode = 'a'
if not add_to_file:
mode = 'w'
if run_index is None:
idx = argsort(self.get_weight_components()[quantity_index])[-1]
else:
idx = run_index
write_to_text_file(filepath, array([variable_list[quantity_index]]), mode=mode)
write_to_text_file(filepath, array([self.get_bias()[quantity_index], self.get_variance()[quantity_index][idx]]),
mode='a', delimiter=' ')
def export_bm_parameters(self, directory, filename=None, quantities=None, run_index=None):
"""Writes BM parameters (bias and variance) for each quantity to a file. The variance parameter is either
the one computed for a run given by run_index, or (if it is not given) for the run with the highest weight.
"""
if filename is None:
filename = 'bm_parameters_' + str(self.get_calibration_year())
if run_index is not None:
filename = '%s_run_%s' % (filename, run_index)
file = os.path.join(directory, filename)
if quantities is None:
variable_list = self.get_variable_names()
else:
variable_list = quantities
write_to_text_file(file, array([self.get_base_year(), self.get_calibration_year()]), delimiter=' ')
for quant in variable_list:
self.export_bm_parameters_of_one_quantity(quant, file, add_to_file=True, run_index=run_index)
def compute_weights(self, procedure="opus_core.bm_normal_weights", **kwargs):
""" Launches the run method of the given 'procedure'. This should return the actual BM weights.
The method passes self as first argument into the run method.
If 'procedure' is not given, the method returns equal weights.
"""
self.compute_y()
self.estimate_mu()
self.estimate_bias()
self.estimate_variance()
if procedure is not None:
procedure_class = ModelComponentCreator().get_model_component(procedure)
self.weights, self.weight_components = procedure_class.run(self, **kwargs)
else:
self.weights = 1.0/self.number_of_runs * ones(self.number_of_runs)
write_to_text_file(os.path.join(self.output_directory, self.weights_file_name),
self.weights)
return self.weights
def export_weights_posterior_mean_and_variance(self, years, quantity_of_interest, directory, filename=None,
use_bias_and_variance_from=None, ids = None, **kwargs):
for year in years:
self.set_posterior(year, quantity_of_interest, use_bias_and_variance_from, **kwargs)
if filename is None:
filename = quantity_of_interest
file = os.path.join(directory, str(year) + '_' + filename)
write_to_text_file(file, concatenate((array([0.]), self.get_weights())), delimiter=' ')
write_to_text_file(file, concatenate((array([0.]), self.get_posterior_component_variance())), mode='a', delimiter=' ')
variable_list = self.get_variable_names()
quantity_index = variable_list.index(quantity_of_interest)
if ids is None:
ids = self.m_ids
means = zeros((ids.size, self.number_of_runs+1))
means[:,0] = ids
means[:,1:means.shape[1]] = self.get_posterior_component_mean()
write_table_to_text_file(file, means, mode='a')
def test_get_node_data_into_node_travel_data_set(self):
temp_dir = tempfile.mkdtemp(prefix='opus_tmp')
file1 = 'report1'
temp_file1 = os.path.join(temp_dir, file1)
write_to_text_file(temp_file1, array(['inode', 'jnode', 'timau', '@corr', 'len', 'result']), delimiter=' ')
write_table_to_text_file(temp_file1, array([[1,2, 35.6, 4, 1.2, 0],
[2,1, 23.5, 3, 0.3,100],
[4,10, 2.1, 3, 0.5, 10],
[3,1, 15.8, 4, 1.1, 5] ]), delimiter = ' ', mode='a')
file2 = 'report2'
temp_file2 = os.path.join(temp_dir, file2)
write_to_text_file(temp_file2, array(['inode', 'jnode', 'volau', 'result']), delimiter=' ')
write_table_to_text_file(temp_file2, array([[1,2, 110, 0],
[3,1, 350, 400],
[5,4, 200, 200]]), delimiter = ' ', mode='a')
node_matrix_attribute_map = {file1: {
'timau':'travel_time',
'len':'distance',
'@corr': 'corridor'
},
file2: {
'volau': 'travel_volume'
}
}
tm_output = TravelModelOutput()
node_travel_data_set = tm_output.get_node_travel_data_set(node_matrix_attribute_map, temp_dir)
# size should be 5, since there are 5 unique combinations of from_node, to_node
self.assertEqual(node_travel_data_set.size(), 5)
# the dataset should have 6 attributes
self.assertEqual(len(node_travel_data_set.get_known_attribute_names()), 6)
self.assertEqual('travel_time' in node_travel_data_set.get_known_attribute_names(), True)
self.assertEqual('distance' in node_travel_data_set.get_known_attribute_names(), True)
self.assertEqual('corridor' in node_travel_data_set.get_known_attribute_names(), True)
self.assertEqual('travel_volume' in node_travel_data_set.get_known_attribute_names(), True)
self.assertEqual('from_node_id' in node_travel_data_set.get_known_attribute_names(), True)
self.assertEqual('to_node_id' in node_travel_data_set.get_known_attribute_names(), True)
# check values of one node
node = node_travel_data_set.get_data_element_by_id((3,1))
self.assertEqual(node.corridor, 4)
self.assertEqual(node.travel_volume, 350)
shutil.rmtree(temp_dir)
def run(self, data, coefficients, resources=None):
"""
Like linear_utilities, but in addition it runs linear utilities for
modified data and stores utilities when each variable is set to its 5%, 95% quantiles,
keeping the other variables at their median. Last row in the resulting file is the difference in
utilities between these two.
The file name can be passed in resources - entry 'utilities_diagnose_file'.
"""
if data.ndim < 3:
raise StandardError, "Argument 'data' must be a 3D numpy array."
if not isinstance(resources, Resources):
resources= Resources(resources)
nobs, neqs, nvar = data.shape
medians = zeros(nvar, dtype=float32)
quant = zeros((2,nvar), dtype=float32)
data_with_medians = array(data[0,:,:])
for ivar in range(nvar): # compute medain and quantiles for each variable
medians[ivar], quant[0,ivar], quant[1,ivar] = quantile(data[:,:,ivar].ravel(), array([0.5, 0.05, 0.95]))
data_with_medians[:,ivar] = medians[ivar]
file_name = resources.get("utilities_diagnose_file", "util")
if resources.get("submodel", None) is not None:
file_name = "%s_submodel_%s" % (file_name, resources.get("submodel", 1))
diagnose_utilities = zeros((3, nvar), dtype=float32)
argcor = ()
for ivar in range(nvar): # iterate over variables
for iquant in [0,1]: # 0 for 5% quantile, 1 for 95% quantile
mod_data = array(data_with_medians).reshape(1,neqs, nvar) # copy original data
mod_data[0,:,ivar] = quant[iquant, ivar]
utility = linear_utilities.run(self, mod_data, coefficients, resources)
diagnose_utilities[iquant, ivar] = utility[0,0]
argcor = argcor + (data[:,:,ivar].ravel(),)
diagnose_utilities[2,:] = diagnose_utilities[1,:] - diagnose_utilities[0,:]
coef_names = resources.get("coefficient_names", map(lambda x: 'x%s' % x, arange(nvar)+1))
write_to_text_file(file_name, coef_names, delimiter=' ')
write_table_to_text_file( file_name, diagnose_utilities, mode='ab')
logger.log_status("Diagnosed utilities written into %s." % file_name)
return linear_utilities.run(self, data, coefficients, resources)
def export_confidence_intervals(self, confidence_levels, filename, delimiter='\t'):
"""Export confidence intervals into a file.
confidence_levels is a list of desired confidence levels.
The resulting file has a id column, 'mean' column and for each level
its lower bound and upper bound columns.
The method generate_posterior_distribution should be called prior to this method.
"""
if not isinstance(confidence_levels, list):
confidence_levels = [confidence_levels]
lcl = len(confidence_levels)
result = zeros((self.simulated_values_ids.size, 2+2*lcl), dtype='float32')
result[:,0] = self.simulated_values_ids
result[:,1] = self.get_quantity_from_simulated_values("mean")
clheader = []
for cl in confidence_levels:
clheader = clheader + ['lower_%s' % cl, 'upper_%s' % cl]
write_to_text_file(filename, ['id', 'mean'] + clheader, delimiter=delimiter)
i = 2
for cl in confidence_levels:
ci = self.get_probability_interval(cl/100.0)
result[:,i:(i+2)] = transpose(ci)
i = i+2
write_table_to_text_file(filename, result, mode='a', delimiter=delimiter)
def write_observed_data(self, filename, index, transformed_back=True):
write_to_text_file(filename,
self.get_observed_data_by_index(
index, transformed_back),
delimiter=' ')
def write_posterior_mean_and_variance(self, mean_filename=None, variance_filename=None):
if mean_filename is not None:
write_table_to_text_file(mean_filename, self.get_posterior_component_mean())
if variance_filename is not None:
write_to_text_file(variance_filename, self.get_posterior_component_variance(), delimiter=' ')
def write_observed_data_for_quantity(self, quantity_of_interest, filename):
data = self.observed_data.get_values_for_quantity(quantity_of_interest)
write_to_text_file(filename, data, delimiter=' ')
def write_observed_data(self, filename, index, transformed_back=True):
write_to_text_file(filename, self.get_observed_data_by_index(index, transformed_back), delimiter=' ')
# Opus/UrbanSim urban simulation software.
# Copyright (C) 2010-2011 University of California, Berkeley, 2005-2009 University of Washington
# See opus_core/LICENSE
from opus_core.services.run_server.generic_option_group import GenericOptionGroup
import os
from numpy import array
from opus_core.misc import write_to_text_file
if __name__ == "__main__":
option_group = GenericOptionGroup()
parser = option_group.parser
parser.add_option("-d", "--cache-directory", dest="cache_directory", action="store", type="string",
help="Cache directory")
parser.add_option("-o", "--output-file", dest="output_file", action="store", type="string",
help="Output file")
(options, args) = parser.parse_args()
files = os.listdir(options.cache_directory)
years = []
for file in files:
try:
year = int(file)
except:
continue
years.append(year)
result = array(years).max()
write_to_text_file(options.output_file, array([result]))
root_seed = config.get("seed", None)
seed(root_seed)
# generate different seed for each run (each seed contains 1 number)
seed_array = randint(1,2**30, number_of_runs)
list_of_cache_directories = []
for irun in range(number_of_runs):
config['seed']= (seed_array[irun],)
this_config = config.copy()
if ((irun + 1) % number_of_runs_in_parallel) == 0:
run_in_background = False
else:
run_in_background = True
run_manager.setup_new_run(cache_directory = this_config['cache_directory'],
configuration = this_config)
run_manager.run_run(this_config, run_as_multiprocess=False,
run_in_background=run_in_background)
if irun == 0:
# log file for the multiple runs will be located in the first cache
first_cache_directory = this_config['cache_directory']
log_file = os.path.join(first_cache_directory, 'multiple_runs.log')
logger.enable_file_logging(log_file)
logger.log_status("Multiple runs: %s replications" % number_of_runs)
logger.log_status("root random seed = %s" % str(root_seed))
else:
logger.enable_file_logging(log_file, verbose=False)
logger.log_status("Run %s: %s" % (irun+1, this_config['cache_directory']))
logger.disable_file_logging(log_file)
list_of_cache_directories.append(this_config['cache_directory'])
write_to_text_file(os.path.join(first_cache_directory,"cache_directories"),
list_of_cache_directories)
for irun in range(number_of_runs):
config['seed'] = (seed_array[irun], )
this_config = config.copy()
if ((irun + 1) % number_of_runs_in_parallel) == 0:
run_in_background = False
else:
run_in_background = True
run_manager.setup_new_run(
cache_directory=this_config['cache_directory'],
configuration=this_config)
run_manager.run_run(this_config,
run_as_multiprocess=False,
run_in_background=run_in_background)
if irun == 0:
# log file for the multiple runs will be located in the first cache
first_cache_directory = this_config['cache_directory']
log_file = os.path.join(first_cache_directory, 'multiple_runs.log')
logger.enable_file_logging(log_file)
logger.log_status("Multiple runs: %s replications" %
number_of_runs)
logger.log_status("root random seed = %s" % str(root_seed))
else:
logger.enable_file_logging(log_file, verbose=False)
logger.log_status("Run %s: %s" %
(irun + 1, this_config['cache_directory']))
logger.disable_file_logging(log_file)
list_of_cache_directories.append(this_config['cache_directory'])
write_to_text_file(
os.path.join(first_cache_directory, "cache_directories"),
list_of_cache_directories)
