def run(self,
specification,
coefficients,
agent_set,
agents_index=None,
**kwargs):
choices = ChoiceModel.run(self,
specification,
coefficients,
agent_set,
agents_index=agents_index,
**kwargs)
if agents_index is None:
agents_index = arange(agent_set.size())
movers_indices = agents_index[where(choices > 0)]
if self.movers_ratio is not None:
n = rint(self.movers_ratio * agents_index.size)
if n < movers_indices.size:
movers_indices = sample_noreplace(movers_indices, n)
# add unplaced agents
unplaced_agents = agents_index[agent_set.get_attribute_by_index(
self.location_id_name, agents_index) <= 0]
logger.log_status(
"%s agents selected by the logit model; %s agents without %s." %
(movers_indices.size, unplaced_agents.size, self.location_id_name))
movers_indices = unique(concatenate((movers_indices, unplaced_agents)))
logger.log_status("Number of movers: " + str(movers_indices.size))
return movers_indices
def write_table(self, table_name, table_data, mode = Storage.OVERWRITE):
"""
'table_name' specifies the subdirectory relative to base directory.
'table_data' is a dictionary where keys are the column names and values
are value arrays of the corresponding columns.
"""
dir = os.path.join(self._get_base_directory(), table_name)
if not os.path.exists(dir):
logger.log_status("%s doesn't exist and is created" % dir)
os.makedirs(dir)
unused_column_size, column_names = self._get_column_size_and_names(table_data)
for column_name in column_names:
col_type = table_data[column_name].dtype.str
column_file = self.storage_file.new_storage_file(column_name, col_type, dir)
existing_files_of_this_name = glob(os.path.join(dir, '%s.*' % column_name))
if (len (existing_files_of_this_name) > 1):
message = "Column '%s' has multiple files with different file extensions:\n" % column_name
for existing_file_name in existing_files_of_this_name:
message += existing_file_name + "\n"
message += "Either the process of copying files into this directory is flawed, or there is a bug in Opus."
raise FltError(message)
if mode == Storage.OVERWRITE:
for existing_file_name in existing_files_of_this_name:
os.remove(existing_file_name)
if mode == Storage.OVERWRITE or len(existing_files_of_this_name) == 0:
table_data[column_name].tofile(column_file.get_name())
def run(self, person_set, household_set, resources=None):
index = AgentRelocationModel.run(self, person_set, resources=resources)
person_ds_name, person_id_name = person_set.get_dataset_name(
), person_set.get_id_name()[0]
hh_ds_name, hh_id_name = person_set.get_dataset_name(
), household_set.get_id_name()[0]
max_hh_id = household_set.get_attribute(hh_id_name).max() + 1
new_hh_id = arange(max_hh_id, max_hh_id + index.size)
person_set.modify_attribute(hh_id_name, new_hh_id, index=index)
household_set.add_elements({hh_id_name: new_hh_id},
require_all_attributes=False)
logger.log_status("%s children leave home and form new %s." %
(index.size, hh_ds_name))
##remove records from household_set that have no persons left
persons = household_set.compute_variables("%s.number_of_agents(%s)" %
(hh_ds_name, person_ds_name),
resources=resources)
index_hh0persons = where(persons == 0)[0]
if index_hh0persons.size > 0:
logger.log_status(
"Removing %s records without %s from %s dataset" %
(index_hh0persons.size, person_ds_name, hh_ds_name))
household_set.remove_elements(index_hh0persons)
def run(self, person_set, household_set, resources=None):
index = AgentRelocationModel.run(self, person_set, resources=resources)
person_ds_name, person_id_name = person_set.get_dataset_name(
), person_set.get_id_name()[0]
hh_ds_name, hh_id_name = person_set.get_dataset_name(
), household_set.get_id_name()[0]
max_person_id = person_set.get_attribute(person_id_name).max() + 1
new_person_id = arange(max_person_id, max_person_id + index.size)
new_born = {}
new_born[person_id_name] = new_person_id
new_born[hh_id_name] = person_set.get_attribute(hh_id_name)[index]
new_born['age'] = ones(index.size, dtype="int32")
##TODO: give better default values
new_born['sex'] = randint(
2, size=index.size) ##TODO: better way to assign sex?
#new_born['education']
#new_born['job_id']
#new_born['race_id']
#new_born['relation']
logger.log_status("Adding %s records to %s dataset" %
(index.size, person_set.get_dataset_name()))
person_set.add_elements(data=new_born,
require_all_attributes=False,
change_ids_if_not_unique=True)
def _convert_lccm_input(self, flt_directory_in, flt_directory_out):
gc.collect()
t1 = time()
lc = LandCoverDataset(in_storage=StorageFactory().get_storage(
'flt_storage', storage_location=flt_directory_in),
out_storage=StorageFactory().get_storage(
'flt_storage',
storage_location=flt_directory_out))
# lc.get_header() # added 23 june 2009 by mm
mask = lc.get_mask()
idx = where(mask == 0)[0]
lcsubset = DatasetSubset(lc, idx)
print "Converting:"
lcsubset.write_dataset(attributes=["relative_x"],
out_table_name="land_covers")
lc.delete_one_attribute("relative_x")
lcsubset.write_dataset(attributes=["relative_y"],
out_table_name="land_covers")
lc.delete_one_attribute("relative_y")
lc.flush_dataset()
gc.collect()
# lc_names = lc.get_primary_attribute_names()
for attr in lc.get_primary_attribute_names():
print " ", attr
lcsubset.write_dataset(attributes=[attr],
out_table_name="land_covers")
lc.delete_one_attribute(attr)
logger.log_status("Data conversion done. " + str(time() - t1) + " s")
def get_travel_data_from_travel_model(self, config, year, zone_set):
"""
"""
logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...')
# print >> sys.stderr, "MATSim replaces only _some_ of the columns of travel_data. Yet, Urbansim does not truly merge them"
# print >> sys.stderr, " but simply overwrites the columns, without looking for a different sequence of from_zone_id, to_zone_id"
# solved 3dec08 by hana
input_directory = os.path.join(os.environ['OPUS_HOME'], "opus_matsim",
"tmp")
logger.log_status("input_directory: " + input_directory)
in_storage = csv_storage(storage_location=input_directory)
table_name = "travel_data"
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=table_name)
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset(in_storage=cache_storage,
in_table_name=table_name)
##TODO:This may not work or may be wrong after the id_names of travel_data
##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang)
existing_travel_data_set.join(
travel_data_set,
travel_data_set.get_non_id_primary_attribute_names(),
metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def demolish_buildings(self, buildings_to_be_demolished, building_dataset, dataset_pool):
if isinstance(buildings_to_be_demolished, list):
buildings_to_be_demolished = array(buildings_to_be_demolished)
if buildings_to_be_demolished.size <= 0:
return
id_index_in_buildings = building_dataset.get_id_index(buildings_to_be_demolished)
parcels = dataset_pool.get_dataset('parcel')
idx_pcl = parcels.get_id_index(unique(building_dataset['parcel_id'][id_index_in_buildings]))
# remove occupants from buildings to be demolished
JAMM = JoinAttributeModificationModel()
for agent_name in ['household', 'job']:
agents = dataset_pool.get_dataset(agent_name)
JAMM.run(agents, building_dataset, index=id_index_in_buildings, value=-1)
building_dataset.remove_elements(id_index_in_buildings)
logger.log_status("%s buildings demolished." % buildings_to_be_demolished.size)
# set land_use_type 'vacant' to parcels with demolished buildings
land_types = dataset_pool.get_dataset('land_use_type')
vac_idx = land_types["land_use_name"] == 'vacant'
if vac_idx.sum() > 0:
code = land_types.get_id_attribute()[vac_idx][0]
nvac = (parcels['land_use_type_id'][idx_pcl] == code).sum()
parcels['land_use_type_id'][idx_pcl] = code
logger.log_status("%s parcels set to vacant." % (idx_pcl.size - nvac))
def chi_square_test_with_known_mean(self,
function,
mean,
variance,
number_of_iterations,
significance_level=0.01,
number_of_tries=5):
"""For each test, run a two-sided Chi^2 test for sigma = sigma_0 vs. sigma != sigma_0, if means are known.
'mean' and 'variance' are arrays whose length must correspond to the array that the given function produces.
Since the stochastic test will fail every once in a while, run the whole
test up to number_of_tries times, until either it succeeds or it fails too many times.
"""
for j in range(number_of_tries):
K = mean.size
x = zeros((number_of_iterations, K), dtype=float32)
for i in range(number_of_iterations):
x[i, :] = function()
stat = (((x - mean)**2.0) / variance).sum()
prob = chisqprob(stat, K * number_of_iterations)
logger.log_status(
"Stochastic Test: Chi^2 test statistic = " + str(stat) +
", df=", str(K * number_of_iterations), ", p=" + str(prob))
if (prob >= significance_level / 2.0) and (
prob <= (1 - significance_level / 2.0)):
# test succeeded -- jump out of the method
return
# test failed more than number_of_tries times
self.fail(
msg="prob=%f is not in [%f,%f]" %
(prob, significance_level / 2.0, 1 - significance_level / 2.0))
def predict(self, predicted_choice_id_name, agents_index=None):
""" Run prediction. Currently makes sense only for choice models."""
# Create temporary configuration where all words 'estimate' are replaced by 'run'
tmp_config = Resources(self.config)
if self.agents_index_for_prediction is None:
self.agents_index_for_prediction = self.get_agent_set_index().copy()
if agents_index is None:
agents_index = self.agents_index_for_prediction
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['coefficients'] = "coeff_est"
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['agents_index'] = "agents_index"
tmp_config['models_configuration'][self.model_name]['controller']['run']['arguments']['chunk_specification'] = "{'nchunks':1}"
### save specification and coefficients to cache (no matter the save_estimation_results flag)
### so that the prepare_for_run method could load specification and coefficients from there
#output_configuration = self.config['output_configuration']
#del self.config['output_configuration']
#self.save_results()
#self.config['output_configuration'] = output_configuration
#self.model_system.run_year_namespace["coefficients"] = self.coefficients
#del tmp_config['models_configuration'][self.model_name]['controller']['prepare_for_run']
try:
run_year_namespace = copy.copy(self.model_system.run_year_namespace)
except:
logger.log_error("The estimate() method must be run first")
return False
try:
agents = self.get_agent_set()
choice_id_name = self.get_choice_set().get_id_name()[0]
# save current locations of agents
current_choices = agents.get_attribute(choice_id_name).copy()
dummy_data = zeros(current_choices.size, dtype=current_choices.dtype)-1
#agents.modify_attribute(name=choice_id_name, data=dummy_data) #reset choices for all agents
agents.modify_attribute(name=choice_id_name, data=dummy_data, index=agents_index) #reset choices for agents in agents_index
run_year_namespace["process"] = "run"
run_year_namespace["coeff_est"] = self.coefficients
run_year_namespace["agents_index"] = agents_index
run_year_namespace["processmodel_config"] = tmp_config['models_configuration'][self.model_name]['controller']['run']
new_choices = self.model_system.do_process(run_year_namespace)
#self.model_system.run(tmp_config, write_datasets_to_cache_at_end_of_year=False)
#new_choices = agents.get_attribute(choice_id_name).copy()
agents.modify_attribute(name=choice_id_name, data=current_choices)
dummy_data[agents_index] = new_choices
if predicted_choice_id_name not in agents.get_known_attribute_names():
agents.add_primary_attribute(name=predicted_choice_id_name, data=dummy_data)
else:
agents.modify_attribute(name=predicted_choice_id_name, data=dummy_data)
logger.log_status("Predictions saved into attribute " + predicted_choice_id_name)
return True
except Exception, e:
logger.log_error("Error encountered in prediction: %s" % e)
logger.log_stack_trace()
def tearDown(self):
print "entering tearDown"
logger.log_status('Removing extracted MATSim files...')
if os.path.exists(self.destination):
rmtree(self.destination)
logger.log_status('... cleaning up finished.')
print "leaving tearDown"
def run(self, year, output_file=None):
"""Runs the emme2 executables, using appropriate info from config.
Assumes the emme2 input files are present.
Raise an exception if the emme2 run fails.
"""
emme2_batch_file_path = self.get_emme2_batch_file_path(year)
emme2_dir, emme2_batch_file_name = os.path.split(emme2_batch_file_path)
logger.log_status('Using emme2 dir %s for year %d' % (emme2_dir, year))
os.chdir(emme2_dir)
if output_file is None:
log_file_path = os.path.join(self.config['cache_directory'], 'emme2_%d_log.txt' % year)
else:
log_file_path = output_file
# if log_file_path is a remote sftp URL, redirect the log file to tempdir
log_file_path = redirect_sftp_url_to_local_tempdir(log_file_path)
cmd = """%(system_cmd)s"%(emme2_batch_file_name)s" > %(log_file_path)s""" % {
'system_cmd': self.config['travel_model_configuration'].get('system_command', 'cmd /c '),
'emme2_batch_file_name':emme2_batch_file_path,
'log_file_path':log_file_path,
}
logger.log_status('Running command %s' % cmd)
cmd_result = os.system(cmd)
if cmd_result != 0:
error_msg = "Emme2 Run failed. Code returned by cmd was %d" % (cmd_result)
logger.log_error(error_msg)
raise StandardError(error_msg)
def test_run(self):
print "Entering test run"
logger.log_status('Loading travel data: %s' % self.travel_data_source)
# get travel data as an attribute marix
in_storage = csv_storage(storage_location=self.travel_data_source_dir)
table_name = "travel_data"
travel_data_attribute = 'single_vehicle_to_work_travel_cost'
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=table_name)
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(
travel_data_attribute, fill=0)
# determine location to store and read attribute matrix
location1 = os.path.join(self.tempDir, 'attrib_matrix1.txt')
location2 = os.path.join(self.tempDir, 'attrib_matrix2.txt')
# store attribute matrix
savetxt(location1, travel_data_attribute_mat, fmt="%i")
savetxt(location2, travel_data_attribute_mat, fmt="%i")
# read attribute matrix
matrix1 = genfromtxt(location1)
matrix2 = genfromtxt(location2)
# compare both matices
result = (matrix1 == matrix2)
self.assertTrue(result.all())
print "Leaving test run"
def run(self, config, year, iteration):
"""
"""
tm_config = config['travel_model_configuration']
# Altering the iteration count in the config file for MALTA
fileLoc = tm_config.get("malta_project")
fileEntries = []
fileMaltaConfig = open(fileLoc, 'r')
for line in fileMaltaConfig:
fileEntries.append(line)
fileMaltaConfig.close()
fileMaltaConfig = open(fileLoc, 'w')
fileMaltaConfig.write(fileEntries[0])
fileMaltaConfig.write('%d\n' %(iteration+1))
fileMaltaConfig.write('-99\n')
fileMaltaConfig.write('-99\n')
fileMaltaConfig.close()
cmd = "%s %s" % (tm_config["malta_path"], tm_config.get("malta_project"))
logger.log_status("Start MALTA...")
try:
stdout, stderr= subprocess.Popen(cmd,
shell = True
).communicate()
except:
print "Error occured when running MALTA"
raise
"""
def get_travel_data_from_travel_model(self, config, year, zone_set):
"""
Returns a new travel data set populated by a travel model
"""
logger.log_status("Running GetTravelModelDataIntoCache with year %d" %
(year))
logger.log_status(zone_set)
tm_config = config['travel_model_configuration']
base_dir = tm_config['travel_model_base_directory']
run_dir = os.path.join(base_dir, tm_config[year]['year_dir'])
in_storage = StorageFactory().get_storage('dict_storage')
max_zone_id = zone_set.get_id_attribute().max()
in_storage.write_table(table_name=self.TABLE_NAME,
table_data=self._read_skims(
run_dir, max_zone_id))
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=self.TABLE_NAME)
# This isn't necessary
# remove_index = where(logical_or(travel_data_set.get_attribute("from_zone_id")>max_zone_id,
# travel_data_set.get_attribute("to_zone_id")>max_zone_id))[0]
# travel_data_set.size()
# travel_data_set.remove_elements(remove_index)
return travel_data_set
def __init__(self, cache_file_location, observed_data,
base_year=0, prefix='run_', package_order=['core'], additional_datasets={},
overwrite_cache_directories_file=False):
""" Class used in the Bayesian melding analysis.
'cache_file_location' is location (either file or directory) with information about all caches
(see doc string for MultipleRuns).
'observed_data' is an object of ObservedData that contains all the information about observed data.
"""
MultipleRuns.__init__(self, cache_file_location, prefix=prefix, package_order=package_order,
additional_datasets=additional_datasets,
overwrite_cache_directories_file=overwrite_cache_directories_file)
self.output_directory = os.path.join(os.path.split(self.full_cache_file_name)[0], 'bm_output')
logger.log_status('Output directory set to %s.' % self.output_directory)
if not os.path.exists(self.output_directory):
os.mkdir(self.output_directory)
self.observed_data = observed_data
self.base_year = base_year
self.propagation_factor = None
self.y = {}
self.mu = {}
self.ahat = {}
self.v = {}
self.weight_components = {}
self.simulated_values = None
self.weights = None
def __init__(self):
print "entering setUp"
logger.log_status('Running MATSim test... ')
# get root path to test cases
self.path = test_path.__path__[0]
logger.log_status('Set root path for MATSim config file to: %s' %
self.path)
if not os.path.exists(self.path):
raise StandardError("Root path doesn't exist: %s" % self.path)
self.source = self.get_matsim_source(
) # get path to MATSim source files
self.destination = tempfile.mkdtemp(prefix='opus_tmp')
#self.destination = '/Users/thomas/Desktop/x' # for debugging
#if not os.path.exists(self.destination): # for debugging
# os.mkdir(self.destination) # for debugging
self.matsim_config_full = os.path.join(self.destination,
"test_matsim_config.xml")
# since pyxb need to be installed this method will be disabled for standard tests ...
# self.create_MATSim_config()
# ... therefore we are coping an exsisting MATSim config file.
self.copy_matsim_config()
print "leaving setUp"
def run_emme2_macro(self,
macro_path,
bank_path,
scenario_number=-1,
output_file=None,
append_to_output=True):
"""
Runs this emme/2 macro in the bank specified.
"""
logger.start_block('Running emme2 macro %s in bank at %s' %
(macro_path, bank_path))
# generate a random file name
temp_macro_file_name = tempfile.NamedTemporaryFile().name
prior_cwd = os.getcwd()
if output_file is None:
out = ""
else:
out = "> %s" % output_file
if append_to_output:
out = " >%s" % out
try:
os.chdir(bank_path)
shutil.copy(macro_path, temp_macro_file_name)
cmd = "%s 000 -m %s" % (self.emme_cmd, temp_macro_file_name)
if scenario_number != -1:
cmd = "%s %s" % (cmd, scenario_number)
cmd = "%s%s" % (cmd, out)
logger.log_status(cmd)
if os.system(cmd):
raise StandardError("Problem with simulation")
finally:
os.remove(temp_macro_file_name)
os.chdir(prior_cwd)
logger.end_block()
def run(self, in_storage, out_storage):
dataset_pool = DatasetPool(
storage=in_storage,
package_order=['psrc_parcel', 'urbanism_parcel', 'urbansim'])
ds = self._do_run(dataset_pool)
logger.log_status("Write building_sqft_per_job table.")
ds.write_dataset(out_storage=out_storage)
def generate_variables_table_sorted_by_module(self, root_module="urbansim", output_path=""):
variable_list = self.variable_modules_sorted(root_module)
module_list = []
while variable_list != []:
module_to_get = variable_list[0][-2]
module_list.append(self.filter_and_remove(lambda x: x[-2] == module_to_get, variable_list))
module_list.sort(lambda x, y: (int)(x[0][-2] > y[0][-2]) - (int)(x[0][-2] < y[0][-2]))
f = open(output_path+"variable_module_tables.html", 'w')
f.write(self.__class__.header)
for module in module_list:
f.write('<h3>'+ module[0][-2] +'</h3>\n')
f.write('<table border="1">\n')
f.write('<tr>\n\t<th>Variable Name</th>\n\t<th>Notes</th>\n</tr>\n')
for name in module:
(doc_string, cls_name) = self.get_doc_string('.'.join(name))
if doc_string == None: doc_string = ''
f.write('<tr>\n\t<td>'+cls_name+
'</td>\n\t<td>'+doc_string +'</td>\n</tr>\n')
f.write('</table>\n<br>\n')
f.write(self.__class__.footer)
f.close()
logger.log_status('html file output completed')
def run_chunk(self, agents_index, agent_set, specification, coefficients):
# unplaced agents in agents_index
location_id_name = self.choice_set.get_id_name()[0]
agent_set.set_values_of_one_attribute(
location_id_name, resize(array([-1]), agents_index.size),
agents_index)
## capacity may need to be re-computed for every chunk
if self.compute_capacity_flag:
self.capacity = ma.filled(
self.determine_capacity(capacity_string=self.run_config.get(
"capacity_string", None),
agent_set=agent_set,
agents_index=agents_index), 0.0)
if self.capacity is not None:
logger.log_status("Available capacity: %s units." %
self.capacity.sum())
self.run_config.merge({"capacity": self.capacity})
if self.run_config.get("agent_units_string", None):
self.run_config[
"agent_units_all"] = agent_set.get_attribute_by_index(
self.run_config["agent_units_string"], agents_index)
choices = ChoiceModel.run_chunk(self, agents_index, agent_set,
specification, coefficients)
## this is done in choice_model
#modify locations
#agent_set.set_values_of_one_attribute(location_id_name, choices, agents_index)
if self.run_config.has_key("capacity"):
del self.run_config["capacity"]
return choices
def _updateConfigPaths(self, configfile, regexlist, doubleBackslash=True):
""" Updates the given configfile with the given regex dictionary.
Regexdict will is a list of tuples: [regex_str,replacement_str]; first match wins
"""
slash_re = re.compile(r"([^\\])\\([^\\])")
# make it a triple: [ regex_str, replacement_str, compiled regex ]
for i in range(len(regexlist)):
regexlist[i].append(re.compile(regexlist[i][0],re.IGNORECASE))
# and make the backslashes double
if doubleBackslash: regexlist[i][1] = slash_re.sub(r"\1\\\\\2", regexlist[i][1])
shutil.move(configfile, configfile+".bak")
infile = open(configfile+".bak", 'rU')
outfile = open(configfile, 'w')
for line in infile:
modified = False
for i in range(len(regexlist)):
(modline, n) = regexlist[i][2].subn(regexlist[i][1], line)
if n > 0:
# forward slash to backslash
modline = modline.replace(r"/", r"\\")
# double the backslashes
if doubleBackslash: modline = slash_re.sub(r"\1\\\\\2", modline)
logger.log_status("Modified %s => %s in %s" %
(line.strip("\n\r"), modline.strip("\n\r"), configfile))
outfile.write(modline)
modified = True
break
if not modified:
outfile.write(line)
infile.close()
outfile.close()
def prepare_for_run(self, configuration_path, run_id_file=None, **kwargs):
self.run_ids_dict = {} # dict of run_id and finished year
if run_id_file is None:
config = get_config_from_opus_path(configuration_path)
self.number_of_runs = config.get("number_of_runs", 1)
root_seed = config.get("seed", None)
seed(root_seed)
# generate different seed for each run (each seed contains 1 number)
seed_array = randint(1, 2**30, self.number_of_runs)
for irun in range(self.number_of_runs):
config['seed'] = (seed_array[irun], )
RemoteRun.prepare_for_run(self,
config=config,
prepare_cache=False)
self.run_ids_dict[self.run_id] = (0, 'NA')
self.run_id_file = self.default_run_id_file
else:
self.read_run_id_file(run_id_file)
self.run_id_file = run_id_file
self.get_run_manager()
self.date_time_str = time.strftime('%Y_%m_%d_%H_%M', time.localtime())
logger.log_status("run_id_file: %s" % self.run_id_file)
self.write_into_run_id_file()
return None
def compare_travel_data_sets(self):
# get copied travel data csv
copied_travel_data_location = os.path.join( self.destination, 'opus_matsim', 'tmp')
if not os.path.exists(copied_travel_data_location):
raise StandardError('Travel data not found: %s' % copied_travel_data_location)
logger.log_status('Get copied travel data: %s' % copied_travel_data_location)
# convert travel data csv into travel data set matrix
in_storage = csv_storage(storage_location = copied_travel_data_location)
table_name = "travel_data"
travel_data_attribute = 'single_vehicle_to_work_travel_cost'
travel_data_set = TravelDataDataset( in_storage=in_storage, in_table_name=table_name )
travel_data_attribute_mat = travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
# get exsisting travel data set and convert it also into travel data set matrix
year = self.run_config['base_year']+2
attribute_cache = AttributeCache(cache_directory=self.run_config['cache_directory'])
cache_storage = attribute_cache.get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset( in_storage=cache_storage, in_table_name=table_name )
existing_travel_data_attribute_mat = existing_travel_data_set.get_attribute_as_matrix(travel_data_attribute, fill=999)
from numpy import savetxt # for debugging
savetxt( os.path.join(self.destination, 'origin_travel_data.txt'), travel_data_attribute_mat , fmt="%f")
savetxt( os.path.join(self.destination, 'existing_travel_data') , existing_travel_data_attribute_mat, fmt="%f")
# compare both data set matices
compare = travel_data_attribute_mat == existing_travel_data_attribute_mat
# return result
return compare.all()
def run(self,
zones,
specification,
coefficients,
agent_set,
agents_index=None,
**kwargs):
if agents_index is None:
agents_index = arange(agent_set.size())
cond_array = zeros(agent_set.size(), dtype="bool8")
cond_array[agents_index] = True
zone_ids = zones.get_id_attribute()
agents_zones = agent_set.get_attribute(zones.get_id_name()[0])
for zone_id in zone_ids:
new_index = where(logical_and(cond_array,
agents_zones == zone_id))[0]
self.filter = "building.zone_id == %s" % zone_id
logger.log_status("HLCM for zone %s" % zone_id)
HouseholdLocationChoiceModel.run(self,
specification,
coefficients,
agent_set,
agents_index=new_index,
**kwargs)
agent_set.flush_dataset()
def run(self, location_set, agent_set, agents_index=None, data_objects=None, **kwargs):
if agents_index is None:
agents_index = arange(agent_set.size())
large_areas = agent_set.get_attribute(self.large_area_id_name)
location_large_area = location_set.compute_variables(["washtenaw.%s.%s" % (location_set.get_dataset_name(), self.large_area_id_name)],
dataset_pool=self.dataset_pool)
valid_large_area = where(large_areas[agents_index] > 0)[0]
if valid_large_area.size > 0:
unique_large_areas = unique(large_areas[agents_index][valid_large_area])
cond_array = zeros(agent_set.size(), dtype="bool8")
cond_array[agents_index[valid_large_area]] = True
for area in unique_large_areas:
new_index = where(logical_and(cond_array, large_areas == area))[0]
self.filter = "%s.%s == %s" % (location_set.get_dataset_name(), self.large_area_id_name, area)
logger.log_status("%s for area %s" % (self.model_short_name, area))
ScalingJobsModel.run(self, location_set, agent_set, agents_index=new_index, **kwargs)
no_large_area = where(large_areas[agents_index] <= 0)[0]
if no_large_area.size > 0: # run the model for jobs that don't have assigned large_area
self.filter = None
logger.log_status("%s for jobs with no area assigned" % self.model_short_name)
choices = ScalingJobsModel.run(self, location_set, agent_set, agents_index=agents_index[no_large_area], **kwargs)
where_valid_choice = where(choices > 0)[0]
choices_index = location_set.get_id_index(choices[where_valid_choice])
chosen_large_areas = location_set.get_attribute_by_index(self.large_area_id_name, choices_index)
agent_set.modify_attribute(name=self.large_area_id_name, data=chosen_large_areas,
index=no_large_area[where_valid_choice])
def create_indicators(self, indicators,
file_name_for_indicator_results = 'indicator_results.html',
display_error_box = False,
show_results = False):
'''Handles the computation of a list of indicators.'''
if (len(indicators) == 0):
return
source_data = indicators[0].source_data
log_file_path = os.path.join(source_data.cache_directory, 'indicators.log')
logger.enable_file_logging(log_file_path, 'a')
logger.log_status('\n%s BEGIN %s %s'
% ('='*29, strftime('%Y_%m_%d_%H_%M', localtime()), '='*29))
for indicator in indicators:
indicator.create(display_error_box = display_error_box)
logger.log_status('%s END %s %s\n'
% ('='*30, strftime('%Y_%m_%d_%H_%M', localtime()), '='*30))
logger.disable_file_logging(log_file_path)
results_page_path = self._write_results(indicators,
source_data,
file_name_for_indicator_results,
display_error_box)
if show_results:
self.show_results(results_page_path)
return results_page_path
def get_travel_data_from_travel_model(self, config, year, zone_set):
"""
"""
logger.log_status('Starting GetMatsimDataIntoCache.get_travel_data...')
input_directory = os.path.join(config['root'], 'opus_matsim', 'tmp')
logger.log_status("input_directory: " + input_directory)
in_storage = csv_storage(storage_location=input_directory)
table_name = "travel_data"
travel_data_set = TravelDataDataset(in_storage=in_storage,
in_table_name=table_name)
cache_storage = AttributeCache().get_flt_storage_for_year(year)
existing_travel_data_set = TravelDataDataset(in_storage=cache_storage,
in_table_name=table_name)
##TODO:This may not work or may be wrong after the id_names of travel_data
##changed from ['from_zone_id', 'to_zone_id'] to _hidden_id (lmwang)
existing_travel_data_set.join(
travel_data_set,
travel_data_set.get_non_id_primary_attribute_names(),
metadata=AttributeType.PRIMARY)
return existing_travel_data_set
def create_building_types_table(self, db_config, db_name):
table_name = 'job_building_types'
dbconfig = DatabaseServerConfiguration(host_name=db_config.host_name,
user_name=db_config.user_name,
protocol='mysql',
password=db_config.password)
db_server = DatabaseServer(dbconfig)
try:
db = db_server.get_database(db_name)
except:
raise NameError, "Unknown database '%s'!" % db_name
logger.log_status('Creating table %s.' % table_name)
try:
db.DoQuery('DROP TABLE IF EXISTS %s;' % table_name)
db.DoQuery('CREATE TABLE %s '
'(id INT, name varchar(50), home_based INT);' %
table_name)
except:
raise NameError, "Invalid table name specified! (%s)" % table_name
db.DoQuery('INSERT INTO %s (id, name, home_based) VALUES'
'(1, "commercial", 0),'
'(3, "industrial", 0),'
'(2, "governmental", 0),'
'(4, "home_based", 1);' % table_name)
def get_travel_data_from_travel_model(self, config,
year, zone_set ):
"""
Returns a new travel data set populated by a travel model
"""
logger.log_status("Running GetTravelModelDataIntoCache with year %d" % (year))
logger.log_status(zone_set)
tm_config = config['travel_model_configuration']
base_dir = tm_config['travel_model_base_directory']
run_dir = os.path.join(base_dir, tm_config[year]['year_dir'])
in_storage = StorageFactory().get_storage('dict_storage')
max_zone_id = zone_set.get_id_attribute().max()
in_storage.write_table(
table_name=self.TABLE_NAME,
table_data=self._read_skims(run_dir, max_zone_id)
)
travel_data_set = TravelDataDataset(in_storage=in_storage, in_table_name=self.TABLE_NAME)
# This isn't necessary
# remove_index = where(logical_or(travel_data_set.get_attribute("from_zone_id")>max_zone_id,
# travel_data_set.get_attribute("to_zone_id")>max_zone_id))[0]
# travel_data_set.size()
# travel_data_set.remove_elements(remove_index)
return travel_data_set
def create_interaction_dataset(self, agent_set, agents_index, config,
*args, **kwargs):
if config is not None and config.get("estimate", False):
id_name = self.choice_set.get_id_name()[0]
mod_id_name = "__%s__" % id_name
if mod_id_name in agent_set.get_known_attribute_names():
agent_set.set_values_of_one_attribute(
id_name, agent_set.get_attribute(mod_id_name))
result = LocationChoiceModel.create_interaction_dataset(
self, agent_set, agents_index, config, **kwargs)
# select randomly buildings to unplace
ntounplace = int(agents_index.size / 4.0)
#ntounplace = 1
#self.dataset_pool.get_dataset("urbansim_constant")["recent_years"])
#idx = sample_noreplace(agents_index, ntounplace)
tmp = randint(0, agents_index.size, ntounplace)
utmp = unique(tmp)
idx = agents_index[utmp]
logger.log_status("Unplace %s buildings." % utmp.size)
if (mod_id_name not in agent_set.get_known_attribute_names()):
agent_set.add_attribute(name=mod_id_name,
data=array(
agent_set.get_attribute(id_name)))
agent_set.set_values_of_one_attribute(id_name, -1.0 * ones(
(idx.size, )), idx)
return result
return LocationChoiceModel.create_interaction_dataset(
self, agent_set, agents_index, config, **kwargs)
def run(self, config, year, *args, **kwargs):
"""Runs the travel model, using appropriate info from config.
"""
tm_config = config["travel_model_configuration"]
self.prepare_for_run(tm_config, year)
project_year_dir = get_project_year_dir(tm_config, year)
# year_dir = tm_config[year] #'CoreEA0511202006\\urbansim\\2001'
# dir_part1,dir_part2 = os.path.split(year_dir)
# while dir_part1:
# dir_part1, dir_part2 = os.path.split(dir_part1)
# project_year_dir = os.path.join(tm_data_dir, dir_part2) #C:/SEMCOG_baseline/CoreEA0511202006
logger.log_status('Start travel model from directory %s for year %d' % (project_year_dir, year))
#for macroname, ui_db_file in tm_config['macro']['run_semcog_travel_model'].iteritems():
#pass
macroname, ui_db_file = tm_config['macro']['run_semcog_travel_model'], tm_config['ui_file']
loops = 1
logger.log_status('Running travel model ...')
tcwcmd = win32api.GetShortPathName(tm_config['transcad_binary'])
os.system('start /B "start TransCAD" %s' % tcwcmd) #start TransCAD in background
time.sleep(1)
#os.system("%s -a %s -ai '%s'" % (tcwcmd, ui_db_file, macroname))
run_transcad_macro(macroname, ui_db_file, loops)
try:
pass
##win32process.TerminateProcess(self.hProcess, 0)
except:
logger.log_warning("The code has problem to terminate the TransCAD it started.")
def __init__(self, model_system, resources, skip_cache_after_each_year=False, log_file_name="run_model_system.log"):
SessionConfiguration(
new_instance=True,
package_order=resources["dataset_pool_configuration"].package_order,
in_storage=AttributeCache(),
)
# logger.enable_memory_logging()
if not resources.get("log_to_stdout", True):
logger.disable_std_out()
profiler = None
if resources.get("profile_filename", None) is not None:
import hotshot
profiler = hotshot.Profile(resources.get("profile_filename"))
write_datasets_to_cache_at_end_of_year = not skip_cache_after_each_year
if profiler is None:
model_system.run(
resources,
write_datasets_to_cache_at_end_of_year=write_datasets_to_cache_at_end_of_year,
log_file_name=log_file_name,
)
else:
profiler.run(
"model_system.run(resources, write_datasets_to_cache_at_end_of_year=write_datasets_to_cache_at_end_of_year, log_file_name=log_file_name)"
)
logger.log_status(
"Profiling data stored in %s. Use the python module hotshot to view them."
% resources.get("profile_filename")
)
profiler.close()
def run(self, year, output_file=None):
"""Runs the daysim executables, using appropriate info from config.
Assumes the daysim input files are present.
Raise an exception if the daysim run fails.
"""
daysim_config_file = self.get_daysim_config_file(year)
daysim_dir = self.get_daysim_dir(year)
logger.log_status('Using DaySim directory %s for year %d' % (daysim_dir, year))
os.chdir(daysim_dir)
if output_file is None:
log_file_path = os.path.join(self.config['cache_directory'], 'daysim_%d_log.txt' % year)
else:
log_file_path = output_file
# if log_file_path is a remote sftp URL, redirect the log file to tempdir
log_file_path = redirect_sftp_url_to_local_tempdir(log_file_path)
cmd = """%(system_cmd)s"%(config_file)s" > %(log_file_path)s""" % {
'system_cmd': self.config['travel_model_configuration'].get('daysim_system_command', 'cmd /c '),
'config_file':daysim_config_file,
'log_file_path':log_file_path,
}
logger.log_status('Running command %s' % cmd)
cmd_result = os.system(cmd)
if cmd_result != 0:
error_msg = "DaySim Run failed. Code returned by cmd was %d" % (cmd_result)
logger.log_error(error_msg)
raise StandardError(error_msg)
def generate_variables_table_sorted_by_module(self,
root_module="urbansim",
output_path=""):
variable_list = self.variable_modules_sorted(root_module)
module_list = []
while variable_list != []:
module_to_get = variable_list[0][-2]
module_list.append(
self.filter_and_remove(lambda x: x[-2] == module_to_get,
variable_list))
module_list.sort(lambda x, y: (int)(x[0][-2] > y[0][-2]) - (int)
(x[0][-2] < y[0][-2]))
f = open(output_path + "variable_module_tables.html", 'w')
f.write(self.__class__.header)
for module in module_list:
f.write('<h3>' + module[0][-2] + '</h3>\n')
f.write('<table border="1">\n')
f.write(
'<tr>\n\t<th>Variable Name</th>\n\t<th>Notes</th>\n</tr>\n')
for name in module:
(doc_string, cls_name) = self.get_doc_string('.'.join(name))
if doc_string == None: doc_string = ''
f.write('<tr>\n\t<td>' + cls_name + '</td>\n\t<td>' +
doc_string + '</td>\n</tr>\n')
f.write('</table>\n<br>\n')
f.write(self.__class__.footer)
f.close()
logger.log_status('html file output completed')
def test_create_tripgen_travel_model_input_files(self):
in_storage = StorageFactory().get_storage(
'sql_storage',
storage_location = self.database)
sc = SessionConfiguration(new_instance=True,
package_order = ['urbansim', 'psrc'],
in_storage=in_storage)
dataset_pool = sc.get_dataset_pool()
TravelModelInputFileWriter().run(self.tempdir_path, 2000, dataset_pool)
logger.log_status('tazdata path: ', self.tempdir_path)
# expected values - data format: {zone:{column_value:value}}
expected_tazdata = {1: [[1,1], [1,2]],
2: [[2,2]],
3: [],
4: [[2,2]]
}
# get real data from file
real_tazdata = {1:[],2:[], 3:[], 4:[]}
# income groups 1 to 4
for i in [1,2,3,4]:
tazdata_file = open(os.path.join(self.tempdir_path, 'tripgen', 'inputtg', 'tazdata.mf9%s' % i), 'r')
for a_line in tazdata_file.readlines():
if a_line[0].isspace():
numbers = a_line.split()
zone_id = int(numbers[0])
job_zone_id = int(numbers[1])
real_tazdata[i].append([zone_id, job_zone_id])
for group in expected_tazdata.keys():
self.assertEqual(real_tazdata[group], expected_tazdata[group],
"income group %d, columns did not match up."%group)
def create_datasets_from_flt(self,
datasets_to_create,
package_name,
additional_arguments={}):
"""
Returns a dictionary containing all of the loaded datasets.
'datasets_to_create' is a dictionary where keys are the dataset names, values are dictionaries
that can contain 'package_name' and 'arguments' (containing arguments to be passed
to the dataset constructor).
additional_arguments is a dictionary specifying additional arguments for the constructor,
e.g. in_storage.
"""
datasets = {}
# load required data from FLT files
logger.log_status("Loading datasets from FLT cache")
args = {}
args.update(additional_arguments)
for dataset_name, dataset_info in datasets_to_create.iteritems():
argstmp = args.copy()
if "arguments" in dataset_info.keys():
argstmp.update(dataset_info["arguments"])
datasets[dataset_name] = self.get_dataset(dataset_name,
package=dataset_info.get(
'package_name',
package_name),
subdir='datasets',
arguments=argstmp)
argstmp.clear()
return datasets
def run(self, location_set, agent_set, agents_index=None, ignore_agents_distribution=False):
"""
'location_set', 'agent_set' are of type Dataset,
'agent_index' are indices of individuals in the agent_set for which
the model runs. If it is None, the whole agent_set is considered.
If ignore_agents_distribution is True, the agents in place are ignored and
the scaling is done proportionally to the weights only.
"""
if agents_index is None:
agents_index=arange(agent_set.size())
if agents_index.size == 0:
logger.log_status('Nothing to be done.')
return array([], dtype='int32')
if self.submodel_string is not None:
submodels = unique(agent_set[self.submodel_string][agents_index])
else:
submodels = [-2]
self.map_agents_to_submodels(submodels, self.submodel_string, agent_set, agents_index,
dataset_pool=self.dataset_pool,
resources = Resources({"debug": self.debug}))
result = array(agents_index.size*[-1], dtype="int32")
if self.observations_mapping['mapped_index'].size == 0:
logger.log_status("No agents mapped to submodels.")
return result
for submodel in submodels:
result[self.observations_mapping[submodel]] = self._simulate_submodel(submodel,
location_set, agent_set, agents_index, ignore_agents_distribution=ignore_agents_distribution).astype(result.dtype)
return result
def _scanForRuns(self):
data_path = self.project.data_path()
if not os.path.exists(data_path):
MessageBox.warning(mainwindow = self.base_widget,
text="Project data path %s doesn't exist. " % data_path + \
"Simulation runs in the Results tab cannot be updated." )
return
run_manager = get_run_manager()
run_manager.clean_runs()
self._sync_base_year_data(run_manager)
run_manager.close()
added_runs, removed_runs = sync_available_runs(self.project)
added_msg = removed_msg = None
if len(added_runs) > 0:
added_msg = ('The following simulation runs have been '
'automatically added to the results manager:\n\n%s'
% '\n'.join(added_runs))
if len(removed_runs) > 0:
removed_msg = ('The following simulation runs have been '
'automatically removed from the results manager:\n\n%s'
% '\n'.join(removed_runs))
if added_msg or removed_msg:
## The idea is to leave the run information to services db & cache, and
## we don't need to save the newly added runs, once we set the temporary
# self.project.dirty = True
text = 'The list of simulation runs has been automatically updated.'
detailed_text = '%s\n%s' % (added_msg or '', removed_msg or '')
logger.log_status(text+'\n'+detailed_text)
def run(self, specification, coefficients, agent_set, agents_index=None, **kwargs):
if agents_index is None:
agents_index = arange(agent_set.size())
large_areas = agent_set.get_attribute(self.large_area_id_name)
self.choice_set.compute_variables(["washtenaw.%s.%s" % (self.choice_set.get_dataset_name(), self.large_area_id_name)],
dataset_pool=self.dataset_pool)
valid_large_area = where(large_areas[agents_index] > 0)[0]
if valid_large_area.size > 0:
unique_large_areas = unique(large_areas[agents_index][valid_large_area])
cond_array = zeros(agent_set.size(), dtype="bool8")
cond_array[agents_index[valid_large_area]] = True
for area in unique_large_areas:
new_index = where(logical_and(cond_array, large_areas == area))[0]
self.filter = "%s.%s == %s" % (self.choice_set.get_dataset_name(), self.large_area_id_name, area)
logger.log_status("ELCM for area %s" % area)
EmploymentLocationChoiceModel.run(self, specification, coefficients, agent_set,
agents_index=new_index, **kwargs)
agent_index_no_large_area = agents_index[ large_areas[agents_index] <= 0 ]
if agent_index_no_large_area.size > 0: # run the ELCM for jobs that don't have assigned large_area
self.filter = None
logger.log_status("ELCM for jobs with no area assigned")
choices = EmploymentLocationChoiceModel.run(self, specification, coefficients, agent_set,
agents_index=agent_index_no_large_area, **kwargs)
where_valid_choice = where(choices > 0)[0]
choices_index = self.choice_set.get_id_index(choices[where_valid_choice])
chosen_large_areas = self.choice_set.get_attribute_by_index(self.large_area_id_name, choices_index)
agent_set.modify_attribute(name=self.large_area_id_name, data=chosen_large_areas,
index=agent_index_no_large_area[where_valid_choice])
def _move_LCCM_converted_directory(self,flt_directory_in,flt_directory_out):
names = os.listdir(flt_directory_in)
logger.log_status("\r\n4. moving converted data\r\n")
for name in names:
if name != "lct.lf4":
shutil.move(os.path.join(flt_directory_in,name),flt_directory_out)
logger.log_status("\t%s successfully moved" % (name))
def run(self, config, show_output = False):
logger.log_status("Caching large SQL tables to: " + config['cache_directory'])
self.show_output = show_output
#import pydevd;pydevd.settrace()
server_configuration = config['scenario_database_configuration']
scenario_database_manager = ScenarioDatabaseManager(
server_configuration = server_configuration,
base_scenario_database_name = server_configuration.database_name
)
self.database_server = DatabaseServer(server_configuration)
database_to_table_mapping = scenario_database_manager.get_database_to_table_mapping()
self.tables_to_cache = config['creating_baseyear_cache_configuration'].tables_to_cache
simulation_state = SimulationState()
if 'low_memory_run' in config:
simulation_state.set_low_memory_run(config['low_memory_run'])
simulation_state.set_cache_directory(config['cache_directory'])
simulation_state.set_current_time(config['base_year'])
self.tables_cached = set()
for database_name, tables in database_to_table_mapping.items():
self.cache_database_tables(config, database_name, tables)
un_cached_tables = set(self.tables_to_cache) - self.tables_cached
if un_cached_tables:
logger.log_warning('The following requested tables were NOT cached:')
for table_name in un_cached_tables:
logger.log_warning('\t%s' % table_name)
def cache_database_table(self, table_name, base_year, database, in_storage, config):
"""Copy this table from input database into attribute cache.
"""
logger.start_block('Caching table %s' % table_name)
try:
#TODO: why is the config being modified...seems like its kind of useless here...
config['storage_location'] = os.path.join(config['cache_directory'], str(base_year), table_name)
if not os.path.exists(config['storage_location']):
flt_storage = StorageFactory().get_storage(
type='flt_storage',
subdir='store',
storage_location=config['storage_location'])
table = database.get_table(table_name)
id_name = [primary_key.name.lower() for primary_key in table.primary_key]
dataset = Dataset(resources=config,
in_storage=in_storage,
out_storage=flt_storage,
in_table_name=table_name,
id_name = id_name)
nchunks = config['creating_baseyear_cache_configuration'].tables_to_cache_nchunks.get(table_name, 1)
current_time = SimulationState().get_current_time()
SimulationState().set_current_time(base_year)
dataset.load_dataset(nchunks=nchunks, flush_after_each_chunk=True)
SimulationState().set_current_time(current_time)
else:
logger.log_status(config['storage_location'] + " already exits; skip caching " + table_name)
finally:
logger.end_block()
def run(self, agent_set,
resources=None,
reset_attribute_value={}):
self.resources.merge(resources)
if agent_set.size()<=0:
agent_set.get_id_attribute()
if agent_set.size()<= 0:
self.debug.print_debug("Nothing to be done.",2)
return array([], dtype='int32')
if self.upc_sequence and (self.upc_sequence.probability_class.rate_set or self.resources.get('rate_set', None)):
self.resources.merge({agent_set.get_dataset_name():agent_set}) #to be compatible with old-style one-relocation_probabilities-module-per-model
self.resources.merge({'agent_set':agent_set})
choices = self.upc_sequence.run(resources=self.resources)
# choices have value 1 for agents that should be relocated, otherwise 0.
movers_indices = where(choices>0)[0]
else:
movers_indices = array([], dtype='int32')
if reset_attribute_value and movers_indices.size > 0:
for key, value in reset_attribute_value.items():
agent_set.modify_attribute(name=key,
data=resize(asarray(value), movers_indices.size),
index=movers_indices)
logger.log_status("Number of agents sampled based on rates: " + str(movers_indices.size))
return movers_indices
def make_database_subset(self, db_server, db, output_db_name, proportion=1.0):
number_of_households = db.GetResultsFromQuery("""select count(*) from households""")[1][0]
number_of_gridcells = db.GetResultsFromQuery("""select count(*) from gridcells""")[1][0]
number_of_gridcells = number_of_gridcells * proportion
commands_to_execute = \
"""drop database if exists %(output_database_name)s ;
create database %(output_database_name)s ; use %(output_database_name)s ;
create table scenario_information as select * from %(input_database_name)s.scenario_information;
create table gridcells as (select * from gridcells order by rand() limit %(number_of_gridcells)i) ;
create index gridcells_grid_id on gridcells (grid_id);
create table households as select hh.* from households as hh, gridcells as gc where hh.grid_id = gc.grid_id;
create table jobs as select j.* from jobs as j, gridcells as gc where j.grid_id = gc.grid_id;
create table development_event_history as select deh.* from
development_event_history as deh, gridcells as gc where deh.grid_id = gc.grid_id;
create table gridcells_in_geography as select gig.* from
gridcells_in_geography as gig, gridcells as gc where gig.grid_id = gc.grid_id;
create index gridcells_zone_id on gridcells (zone_id);
create table zones as select distinct z.*
from zones as z where z.zone_id in (select distinct zone_id from gridcells);
create index zones_zone_id on zones (zone_id);
create table travel_data1 as select td.*
from travel_data as td, zones where td.from_zone_id = zones.zone_id;
create table travel_data as select td.* from travel_data1 as td, zones where td.to_zone_id = zones.zone_id;
drop table travel_data1;
create table annual_employment_control_totals as select * from annual_employment_control_totals;
update annual_employment_control_totals set total_home_based_employment =
total_home_based_employment * ((select count(*) from jobs) / (select count(*) from jobs));
update annual_employment_control_totals set total_non_home_based_employment =
total_non_home_based_employment * ((select count(*) from jobs) / (select count(*) from jobs));
create table annual_household_control_totals as select * from annual_household_control_totals;
update annual_household_control_totals set total_number_of_households = total_number_of_households * (
(select count(*) from households) / (select count(*) from households))""" % \
{"output_database_name":output_db_name,
"input_database_name":db.database_name,
"number_of_gridcells": number_of_gridcells}
commands_to_execute = commands_to_execute.replace('\n', ' ')
command_list = str.split(commands_to_execute, ';')
for command in command_list:
command = command.strip()
logger.log_status(command)
db_server.execute(command)
logger.log_status(output_db_name + ' created successfully!')
def run(self, config, year):
""" This class simulates a MATSim run copying a manipulated
travel data into opus home tmp directory
"""
logger.start_block("Starting CopyDummyTravelData.run(...)")
self.config = config
# get travel model parameter from the opus dictionary
self.travel_model_configuration = config['travel_model_configuration']
self.base_year = self.travel_model_configuration['base_year']
# for debugging
#try: #tnicolai
# import pydevd
# pydevd.settrace()
#except: pass
# set travel data for test simulation
if year == self.base_year+1:
logger.log_status("Prepare copying pre-calculated MATSim travel data to OPUS_HOME tmp directory.")
self.copy_dummy_travel_data()
self.modify_workplace_accessibility()
# use modified travel data for all following runs
else:
logger.log_status("Travel data is already copied in the first iteration.")
logger.end_block()
def write_table(self, table_name, table_data, mode = Storage.APPEND, table_meta={}, column_meta={}, driver=None, **kwargs):
"""
table_data is a dictionary where keys are the column names and values
are value arrays of the corresponding columns.
Each table is stored as a group called table_name. Each column is stored as an hdf5 dataset.
By default, data are appended to an existing file. Set the argument mode to 'o' or 'w' to overwrite the file.
Meta data for the whole table and/or for the columns can be passed as dictionaries table_meta and column_meta, respectively.
Keys in column_meta have to correspond to column names. The values are again dictionaries with the meta data
which can be either strings, scalar or arrays (this is hdf5 restriction).
Argument driver is passed to the h5py.File. Other arguments can be passed to the h5py create_dataset function,
e.g. compression.
"""
file_name = self.get_storage_location()
dir = os.path.split(file_name)[0]
if len(dir) > 0 and not os.path.exists(dir):
logger.log_status("%s doesn't exist and is created" % dir)
os.makedirs(dir)
unused_column_size, column_names = self._get_column_size_and_names(table_data)
h5mode = self._get_hdf5mode(mode)
f = h5py.File(file_name, h5mode, driver=driver)
if h5mode == 'a' and table_name in f.keys():
raise StandardError, 'File %s already contains table %s. Use mode="w" to overwrite the table.' % (file_name, table_name)
group = f.create_group(table_name)
self._write_columns(group, column_names, table_data, table_meta, column_meta, **kwargs)
f.close()
def _call_input_file_writer(self, year, datasets, in_table_names, out_table_names, variable_names, dataset_pool):
current_year_tm_dir = self.get_daysim_dir(year)
file_config = self.config['travel_model_configuration'].get('daysim_file', {})
file_format = file_config.get('format', 'tab')
if(file_format == 'hdf5g'):
current_year_tm_dir = os.path.join(current_year_tm_dir, file_config.get('name', 'daysim_inputs.hdf5'))
meta_data = self.config['travel_model_configuration'].get('meta_data', {})
storage = StorageFactory().get_storage('%s_storage' % file_format, storage_location = current_year_tm_dir)
kwargs = {}
mode={file_format: Storage.OVERWRITE}
if file_format == 'csv' or file_format == 'tab' or file_format == 'tsv' or file_format == 'dat':
kwargs['append_type_info'] = False
if file_format.startswith('hdf5'):
kwargs['compression'] = file_config.get('hdf5_compression', None)
logger.start_block('Writing Daysim inputs.')
for dataset_name, dataset in datasets.iteritems():
ds_meta = meta_data.get(in_table_names[dataset_name], {})
if file_format.startswith('hdf5'):
kwargs['column_meta'] = ds_meta
attr_vals = {}
for attr in variable_names[dataset_name]:
attr_vals[attr] = dataset[attr]
storage.write_table(table_name = out_table_names[dataset_name], table_data = attr_vals, mode = mode[file_format], **kwargs)
mode['hdf5g'] = Storage.APPEND
logger.end_block()
logger.log_status('Daysim inputs written into %s' % current_year_tm_dir)
return out_table_names.values()
def delete_table(self, table_name):
dir = os.path.join(self._get_base_directory(), table_name)
if os.path.exists(dir):
shutil.rmtree(dir)
logger.log_status("Table %s successfully deleted." % dir)
return True
return False
def _invoke_with_paramiko_wait_until_done_or_failed(self, ssh_client, run_id=None, raise_at_error=False, msg='\n'):
stdin, stdout, stderr = ssh_client.ssh.exec_command(cmd)
while True:
#handle when command returns an error in stderr
try:
stdout_msg = stdout.readlines()
except:
stdout_msg = ''
try:
stderr_msg = stderr.readlines()
except:
stderr_msg = ''
if len(stderr_msg) > 0:
logger.log_error('[' + time.ctime + '] ' + "Error encountered executing cmd through ssh:\n" + ''.join(stderr_msg))
if raise_at_error:
raise RuntimeError, "Error encountered executing cmd through ssh:\n" + ''.join(stderr_msg)
if len(stdout_msg) > 0:
logger.log_status('[' + time.ctime + '] ' + 'stdout:' + ''.join(stdout_msg))
if run_id:
runs_by_status = self.get_run_manager().get_runs_by_status([run_id])
if run_id in runs_by_status.get('done', []):
break
if run_id in runs_by_status.get('failed', []):
raise RuntimeError, "run failed: %s." % msg
time.sleep(60)
def run_emme2_macro(self, macro_path, bank_path, scenario_number=-1, output_file=None, append_to_output=True):
"""
Runs this emme/2 macro in the bank specified.
"""
logger.start_block('Running emme2 macro %s in bank at %s' %
(macro_path, bank_path))
# generate a random file name
temp_macro_file_name = tempfile.NamedTemporaryFile().name
prior_cwd = os.getcwd()
if output_file is None:
out = ""
else:
out = "> %s" % output_file
if append_to_output:
out = " >%s" % out
try:
os.chdir(bank_path)
shutil.copy(macro_path, temp_macro_file_name)
cmd = "%s 000 -m %s" % (self.emme_cmd, temp_macro_file_name)
if scenario_number != -1:
cmd = "%s %s" % (cmd, scenario_number)
cmd = "%s%s" % (cmd, out)
logger.log_status(cmd)
if os.system(cmd):
raise StandardError("Problem with simulation")
finally:
os.remove(temp_macro_file_name)
os.chdir(prior_cwd)
logger.end_block()
def run(self, year, job_set, control_totals, job_building_types, data_objects=None, resources=None):
self._do_initialize_for_run(job_set, job_building_types, data_objects)
large_area_ids = control_totals.get_attribute("large_area_id")
jobs_large_area_ids = job_set.compute_variables("washtenaw.job.large_area_id")
unique_large_areas = unique(large_area_ids)
is_year = control_totals.get_attribute("year")==year
all_jobs_index = arange(job_set.size())
sectors = unique(control_totals.get_attribute("sector_id")[is_year])
self._compute_sector_variables(sectors, job_set)
for area in unique_large_areas:
idx = where(logical_and(is_year, large_area_ids == area))[0]
self.control_totals_for_this_year = DatasetSubset(control_totals, idx)
jobs_index = where(jobs_large_area_ids == area)[0]
jobs_for_this_area = DatasetSubset(job_set, jobs_index)
logger.log_status("ETM for area %s (currently %s jobs)" % (area, jobs_for_this_area.size()))
last_remove_idx = self.remove_jobs.size
self._do_run_for_this_year(jobs_for_this_area)
add_jobs_size = self.new_jobs[self.location_id_name].size-self.new_jobs["large_area_id"].size
remove_jobs_size = self.remove_jobs.size-last_remove_idx
logger.log_status("add %s, remove %s, total %s" % (add_jobs_size, remove_jobs_size,
jobs_for_this_area.size()+add_jobs_size-remove_jobs_size))
self.new_jobs["large_area_id"] = concatenate((self.new_jobs["large_area_id"],
array(add_jobs_size*[area], dtype="int32")))
# transform indices of removing jobs into indices of the whole dataset
self.remove_jobs[last_remove_idx:self.remove_jobs.size] = all_jobs_index[jobs_index[self.remove_jobs[last_remove_idx:self.remove_jobs.size]]]
self._update_job_set(job_set)
idx_new_jobs = arange(job_set.size()-self.new_jobs["large_area_id"].size, job_set.size())
jobs_large_area_ids = job_set.compute_variables("washtenaw.job.large_area_id")
jobs_large_area_ids[idx_new_jobs] = self.new_jobs["large_area_id"]
job_set.delete_one_attribute("large_area_id")
job_set.add_attribute(jobs_large_area_ids, "large_area_id", metadata=AttributeType.PRIMARY)
# return an index of new jobs
return arange(job_set.size()-self.new_jobs["large_area_id"].size, job_set.size())
def write_table(self, table_name, table_data, mode = Storage.OVERWRITE):
"""
'table_name' specifies the subdirectory relative to base directory.
'table_data' is a dictionary where keys are the column names and values
are value arrays of the corresponding columns.
"""
dir = os.path.join(self._get_base_directory(), table_name)
if not os.path.exists(dir):
logger.log_status("%s doesn't exist and is created" % dir)
os.makedirs(dir)
unused_column_size, column_names = self._get_column_size_and_names(table_data)
for column_name in column_names:
col_type = table_data[column_name].dtype.str
column_file = self.storage_file.new_storage_file(column_name, col_type, dir)
existing_files_of_this_name = glob(os.path.join(dir, '%s.*' % column_name))
if (len (existing_files_of_this_name) > 1):
message = "Column '%s' has multiple files with different file extensions:\n" % column_name
for existing_file_name in existing_files_of_this_name:
message += existing_file_name + "\n"
message += "Either the process of copying files into this directory is flawed, or there is a bug in Opus."
raise FltError(message)
if mode == Storage.OVERWRITE:
for existing_file_name in existing_files_of_this_name:
os.remove(existing_file_name)
if mode == Storage.OVERWRITE or len(existing_files_of_this_name) == 0:
table_data[column_name].tofile(column_file.get_name())
def compare_and_try_raise_coeflengthexception(self, value, compvalue, name):
if value != compvalue:
try:
raise CoefLengthException(name)
except CoefLengthException, msg:
logger.log_status(msg)
sys.exit(1)
def delete_table(self, table_name):
dir = os.path.join(self._get_base_directory(), table_name)
if os.path.exists(dir):
shutil.rmtree(dir)
logger.log_status("Table %s successfully deleted." % dir)
return True
return False
def create_building_types_table(self, db_config, db_name):
table_name = 'job_building_types'
dbconfig = DatabaseServerConfiguration(
host_name = db_config.host_name,
user_name = db_config.user_name,
protocol = 'mysql',
password = db_config.password
)
db_server = DatabaseServer(dbconfig)
try:
db = db_server.get_database(db_name)
except:
raise NameError, "Unknown database '%s'!" % db_name
logger.log_status('Creating table %s.' % table_name)
try:
db.DoQuery('DROP TABLE IF EXISTS %s;' % table_name)
db.DoQuery('CREATE TABLE %s '
'(id INT, name varchar(50), home_based INT);'
% table_name)
except:
raise NameError, "Invalid table name specified! (%s)" % table_name
db.DoQuery('INSERT INTO %s (id, name, home_based) VALUES'
'(1, "commercial", 0),'
'(3, "industrial", 0),'
'(2, "governmental", 0),'
'(4, "home_based", 1);'
% table_name)
def invoke_run_travel_model(config, year):
"""
"""
tm_config = config['travel_model_configuration']
scenario = tm_config['travel_model_scenario']
try:
travel_model_year = travel_model_year_mapping[year]
except KeyError:
logger.log_warning("no travel model year mapping for %d." % year)
travel_model_year = year
# put the travel model input data in place
data_exchange_dir = mtc_common.tm_get_data_exchange_dir(config, year)
cache_directory = config['cache_directory']
mtc_dir = os.path.join(cache_directory, "mtc_data")
for f in glob.glob(os.path.join(mtc_dir, '*' + str(year) + '*')):
logger.log_status("Copying over travel model input " + f + " to " + data_exchange_dir)
shutil.copy(f, data_exchange_dir)
my_location = os.path.split(__file__)[0]
script_filepath = os.path.join(my_location, "run_travel_model.py")
cmd = "%s %s -s %s -y %s -n" % (sys.executable, script_filepath, scenario, travel_model_year)
# form the desired output dir for the travel model data. Make it look
# like the urbansim run cache for easy association. Note that we
# explicitly use the forward slash instead of os.sep and friends
# because the travel model is managed via ssh on a cygwin machine, not
# run on the local machine.
outdir = "runs/" + config['cache_directory'].split(os.sep)[-1]
outdir = outdir + "/%d_%s" % (year, scenario)
cmd = cmd + " -o " + outdir
logger.log_status("Launching %s" % cmd)
if os.system(cmd) != 0:
raise TravelModelError
# Run the emfac report
# TODO: the travel model server should come from the configuration. But
# for this we must migrate the configuration from mtc_config.py to the
# top-level xml. So for now we just hard-code it :-/ Same for
# travel_model_home.
tm_server = "*****@*****.**"
travel_model_home = "/cygdrive/e/mtc_travel_model/"
server_model = winssh.winssh(tm_server, "OPUS_MTC_SERVER_PASSWD")
(rc, emfac_windir) = server_model.cmd("cygpath -w " + outdir)
if rc != 0:
logger.log_error("Failed to find windows path for emfac dir " + outdir)
sys.exit(1)
emfac_windir = emfac_windir.replace('\r', '').replace('\n','')
logger.log_status("Attempting to generate EMFAC report...")
cmd = 'cd ' + travel_model_home + 'model_support_files/EMFAC_Files'
logger.log_status(cmd)
server_model.cmd_or_fail(cmd)
cmd = "cmd /c 'RunEmfac.bat " + emfac_windir + " " + str(year) + "' | tee emfac.log"
logger.log_status(cmd)
(rc, out) = server_model.cmd(cmd, supress_output=False, pipe_position=0)
if rc != 0:
logger.log_warning("WARNING: Failed to prepare emfac report")
def run(self,
year,
job_set,
control_totals,
job_building_types,
data_objects=None,
resources=None):
self._do_initialize_for_run(job_set, job_building_types, data_objects)
subarea_ids = control_totals.get_attribute(self.subarea_id_name)
jobs_subarea_ids = job_set.compute_one_variable_with_unknown_package(
variable_name="%s" % (self.subarea_id_name),
dataset_pool=self.dataset_pool)
unique_subareas = unique(subarea_ids)
is_year = control_totals.get_attribute("year") == year
all_jobs_index = arange(job_set.size())
sectors = unique(control_totals.get_attribute("sector_id")[is_year])
self._compute_sector_variables(sectors, job_set)
for area in unique_subareas:
idx = where(logical_and(is_year, subarea_ids == area))[0]
self.control_totals_for_this_year = DatasetSubset(
control_totals, idx)
jobs_index = where(jobs_subarea_ids == area)[0]
jobs_for_this_area = DatasetSubset(job_set, jobs_index)
logger.log_status("ETM for area %s (currently %s jobs)" %
(area, jobs_for_this_area.size()))
last_remove_idx = self.remove_jobs.size
self._do_run_for_this_year(jobs_for_this_area)
add_jobs_size = self.new_jobs[
self.location_id_name].size - self.new_jobs[
self.subarea_id_name].size
remove_jobs_size = self.remove_jobs.size - last_remove_idx
logger.log_status(
"add %s, remove %s, total %s" %
(add_jobs_size, remove_jobs_size,
jobs_for_this_area.size() + add_jobs_size - remove_jobs_size))
self.new_jobs[self.subarea_id_name] = concatenate(
(self.new_jobs[self.subarea_id_name],
array(add_jobs_size * [area], dtype="int32")))
# transform indices of removing jobs into indices of the whole dataset
self.remove_jobs[last_remove_idx:self.remove_jobs.
size] = all_jobs_index[jobs_index[
self.remove_jobs[last_remove_idx:self.
remove_jobs.size]]]
self._update_job_set(job_set)
idx_new_jobs = arange(
job_set.size() - self.new_jobs[self.subarea_id_name].size,
job_set.size())
jobs_subarea_ids = job_set.compute_one_variable_with_unknown_package(
variable_name="%s" % (self.subarea_id_name),
dataset_pool=self.dataset_pool)
jobs_subarea_ids[idx_new_jobs] = self.new_jobs[self.subarea_id_name]
job_set.delete_one_attribute(self.subarea_id_name)
job_set.add_attribute(jobs_subarea_ids,
self.subarea_id_name,
metadata=AttributeType.PRIMARY)
# return an index of new jobs
return arange(
job_set.size() - self.new_jobs[self.subarea_id_name].size,
job_set.size())
def map_agents_to_submodels(self,
submodels,
submodel_string,
agent_set,
agents_index,
dataset_pool=None,
resources=None,
submodel_size_max=None):
""" Creates a class attribute self.observations_mapping which is a dictionary
where each entry corresponds to one submodel. It contains indices
of agents (within agents_index) that belong to that submodel.
Additionally, self.observations_mapping has an entry 'index' which contains agents_index, and an entry
'mapped_index' which contains only indices of agents_index that are included in any of the submodel entries of
observations_mapping. Thus, missing entries of 'index' are agents that do not belong to any submodel.
'submodels' is a list of submodels to be considered.
'submodel_string' specifies the name of attribute/variable that distinguishes submodels.
'resources' are passed to the computation of variable 'submodel_string'.
'submodel_size_max' determines the maximum size of a submodel. If the actual size exceeds this number,
agents are randomly sampled so that the submodel size matches this number.
"""
self.observations_mapping = {
} # maps to which submodel each observation belongs to
nsubmodels = len(submodels)
if (nsubmodels > 1) or ((nsubmodels == 1) and (submodels[0] <> -2)):
try:
agent_set.compute_variables(submodel_string,
dataset_pool=dataset_pool,
resources=resources)
except:
pass
submodel_alias = None
if submodel_string is not None:
submodel_alias = VariableName(submodel_string).get_alias()
if (nsubmodels == 1) and (
(submodel_string is None) or
(submodel_alias not in agent_set.get_known_attribute_names())):
self.observations_mapping[submodels[0]] = arange(
agents_index.size)
else:
for submodel in submodels: #mapping agents to submodels
w = where(
agent_set.get_attribute_by_index(
submodel_alias, agents_index) == submodel)[0]
if submodel_size_max is not None and submodel_size_max < w.size:
# sub-sample from submodel
wnew = sample_noreplace(w, submodel_size_max)
logger.log_status(
'Number of records in submodel %s reduced from %s to %s.'
% (submodel, w.size, wnew.size))
w = sort(wnew)
self.observations_mapping[submodel] = w
else: # no submodel distinction
self.observations_mapping[-2] = arange(agents_index.size)
mapped = zeros(agents_index.size, dtype='bool8')
for submodel, index in self.observations_mapping.iteritems():
mapped[index] = True
self.observations_mapping["index"] = agents_index
self.observations_mapping["mapped_index"] = where(mapped)[0]
def compare_and_try_raise_coeflengthexception(self, value, compvalue,
name):
if value != compvalue:
try:
raise CoefLengthException(name)
except CoefLengthException, msg:
logger.log_status(msg)
sys.exit(1)
def test_sample_replace(self):
start_time = time.time()
sample = sample_replace(self.all, self.size, return_index=True)
logger.log_status("sample_replace %s from %s items array in " % (self.size,self.n) + str(time.time() - start_time) + " sec")
self.assertEqual(sample.size, self.size, msg ="sample size not equal to size parameter")
assert isinstance(sample, ndarray), "sample is not of type ndarray"
assert 0 <= sample.min() <= self.n-1, "sampled elements not in between min and max of source array"
assert 0 <= sample.max() <= self.n-1, "sampled elements not in between min and max of source array"
