def __init__(self, probabilities="urbansim.rate_based_probabilities", choices="opus_core.random_choices", location_id_name="grid_id", model_name="Agent Relocation Model", debuglevel=0, resources=None): self.model_name = model_name self.location_id_name = location_id_name self.debug = DebugPrinter(debuglevel) self.upc_sequence = None if probabilities is not None: self.upc_sequence = UPCFactory().get_model( utilities=None, probabilities=probabilities, choices=choices, debuglevel=debuglevel) self.resources = merge_resources_if_not_None(resources)
def __init__(self, probabilities = "urbansim.rate_based_probabilities", choices = "opus_core.random_choices", location_id_name="grid_id", model_name = "Agent Relocation Model", debuglevel=0, resources=None ): self.model_name = model_name self.location_id_name = location_id_name self.debug = DebugPrinter(debuglevel) self.upc_sequence = None if probabilities is not None: self.upc_sequence = UPCFactory().get_model(utilities=None, probabilities=probabilities, choices=choices, debuglevel=debuglevel) self.resources = merge_resources_if_not_None(resources)
def run(self, data, upc_sequence, resources=None): self.mnl_probabilities=upc_sequence.probability_class self.bhhh_estimation = bhhh_mnl_estimation() modified_upc_sequence = UPCFactory().get_model( utilities=None, probabilities="opus_core.mnl_probabilities", choices=None) modified_upc_sequence.utility_class = upc_sequence.utility_class N, neqs, V = data.shape max_iter = resources.get("max_iterations", 100) # default sc = SessionConfiguration() dataset_pool = sc.get_dataset_pool() sample_rate = dataset_pool.get_dataset("sample_rate") CLOSE = sc["CLOSE"] info_filename = sc["info_file"] info_filename = os.path.join('.', info_filename) info_file = open(info_filename, "a") constraint_dict = {1:'constrained', 0:'unconstrained'} swing_cases_fix = 0 #set swing alternatives to constrained (1) or unconstrained (0) prob_correlation = None choice_set = resources['_model_'].choice_set J = choice_set.size() alt_id = choice_set.get_id_attribute() movers = choice_set.get_attribute('movers') resources.check_obligatory_keys(["capacity_string"]) supply = choice_set.get_attribute(resources["capacity_string"]) index = resources.get("index", None) if index is None: # no sampling case, alternative set is the full choice_set index = arange(J) if index.ndim <= 1: index = repeat(index[newaxis,:], N, axis=0) if resources.get('aggregate_to_dataset', None): aggregate_dataset = dataset_pool.get_dataset(resources.get('aggregate_to_dataset')) choice_set_aggregate_id = choice_set.get_attribute(aggregate_dataset.get_id_name()[0]) index = aggregate_dataset.get_id_index(choice_set_aggregate_id[index].ravel()).reshape(index.shape) supply = aggregate_dataset.get_attribute(resources["capacity_string"]) J = aggregate_dataset.size() movers = aggregate_dataset.get_attribute("movers") demand_history = movers[:, newaxis] resources.merge({"index":index}) pi = ones(index.shape, dtype=float32) #initialize pi #average_omega = ones(J,dtype=float32) #initialize average_omega logger.start_block('Outer Loop') for i in range(max_iter): logger.log_status('Outer Loop Iteration %s' % i) result = self.bhhh_estimation.run(data, modified_upc_sequence, resources) del self.bhhh_estimation; collect() self.bhhh_estimation = bhhh_mnl_estimation() probability = modified_upc_sequence.get_probabilities() if data.shape[2] == V: #insert a placeholder for ln(pi) in data data = concatenate((data,ones((N,neqs,1),dtype=float32)), axis=2) coef_names = resources.get("coefficient_names") coef_names = concatenate( (coef_names, array(["ln_pi"])) ) resources.merge({"coefficient_names":coef_names}) else: beta_ln_pi = result['estimators'][where(coef_names == 'ln_pi')][0] logger.log_status("mu = 1/%s = %s" % (beta_ln_pi, 1/beta_ln_pi)) prob_hat = safe_array_divide(probability, pi ** beta_ln_pi) #prob_hat = safe_array_divide(probability, pi) prob_hat_sum = prob_hat.sum(axis=1, dtype=float32) if not ma.allclose(prob_hat_sum, 1.0): logger.log_status("probability doesn't sum up to 1, with minimum %s, and maximum %s" % (prob_hat_sum.min(), prob_hat_sum.max())) probability = normalize(prob_hat) demand = self.mnl_probabilities.get_demand(index, probability, J) * 1 / sample_rate demand_history = concatenate((demand_history, demand[:, newaxis]), axis=1) sdratio = safe_array_divide(supply, demand, return_value_if_denominator_is_zero=2.0) sdratio_matrix = sdratio[index] ## debug info from numpy import histogram from opus_core.misc import unique cc = histogram(index.ravel(), unique(index.ravel()))[0] logger.log_status( "=================================================================") logger.log_status( "Probability min: %s, max: %s" % (probability.min(), probability.max()) ) logger.log_status( "Demand min: %s, max: %s" % (demand.min(), demand.max()) ) logger.log_status( "sdratio min: %s, max: %s" % (sdratio.min(), sdratio.max()) ) logger.log_status( "demand[sdratio==sdratio.min()]=%s" % demand[sdratio==sdratio.min()] ) logger.log_status( "demand[sdratio==sdratio.max()]=%s" % demand[sdratio==sdratio.max()] ) logger.log_status( "Counts of unique submarkets in alternatives min: %s, max: %s" % (cc.min(), cc.max()) ) logger.log_status( "=================================================================") constrained_locations_matrix, omega, info = self.inner_loop(supply, demand, probability, index, sdratio_matrix, J, max_iteration=max_iter) inner_iterations, constrained_locations_history, swing_index, average_omega_history = info for idx in swing_index: logger.log_status("swinging alt with id %s set to %s" % (alt_id[idx], constraint_dict[swing_cases_fix])) constrained_locations_matrix[index==idx] = swing_cases_fix if swing_index.size > 0: info_file.write("swing of constraints found with id %s \n" % alt_id[swing_index]) info_file.write("outer_iteration, %i, " % i + ", ".join([str(i)]*(len(inner_iterations))) + "\n") info_file.write("inner_iteration, , " + ", ".join(inner_iterations) + "\n") info_file.write("id, sdratio, " + ", ".join(["avg_omega"]*len(inner_iterations)) + "\n") for idx in swing_index: line = str(alt_id[idx]) + ',' line += str(sdratio[idx]) + ',' line += ",".join([str(x) for x in average_omega_history[idx,]]) line += "\n" info_file.write(line) info_file.write("\n") info_file.flush() outer_iterations = [str(i)] * len(inner_iterations) prob_min = [str(probability.min())] * len(inner_iterations) prob_max = [str(probability.max())] * len(inner_iterations) pi_new = self.mnl_probabilities.get_pi(sdratio_matrix, omega, constrained_locations_matrix) data[:,:,-1] = ln(pi_new) #diagnostic output if not ma.allclose(pi, pi_new, atol=CLOSE): if i > 0: #don't print this for the first iteration logger.log_status("min of abs(pi(l+1) - pi(l)): %s" % absolute(pi_new - pi).min()) logger.log_status("max of abs(pi(l+1) - pi(l)): %s" % absolute(pi_new - pi).max()) logger.log_status("mean of pi(l+1) - pi(l): %s" % (pi_new - pi).mean()) logger.log_status('Standard Deviation pi(l+1) - pi(l): %s' % standard_deviation(pi_new - pi)) logger.log_status('correlation of pi(l+1) and pi(l): %s' % corr(pi_new.ravel(), pi.ravel())[0,1]) pi = pi_new probability_old = probability # keep probability of the previous loop, for statistics computation only else: #convergence criterion achieved, quiting outer loop logger.log_status("pi(l) == pi(l+1): Convergence criterion achieved") info_file.write("\nConstrained Locations History:\n") info_file.write("outer_iteration," + ",".join(outer_iterations) + "\n") info_file.write("inner_iteration," + ",".join(inner_iterations) + "\n") info_file.write("minimum_probability," + ",".join(prob_min) + "\n") info_file.write("maximum_probability," + ",".join(prob_max) + "\n") for row in range(J): line = [str(x) for x in constrained_locations_history[row,]] info_file.write(str(alt_id[row]) + "," + ",".join(line) + "\n") info_file.flush() info_file.write("\nDemand History:\n") i_str = [str(x) for x in range(i)] info_file.write("outer_iteration, (movers)," + ",".join(i_str) + "\n") #info_file.write(", ,\n") for row in range(J): line = [str(x) for x in demand_history[row,]] info_file.write(str(alt_id[row]) + "," + ",".join(line) + "\n") demand_history_info_criteria = [500, 100, 50, 20] for criterion in demand_history_info_criteria: com_rows_index = where(movers <= criterion)[0] info_file.write("\nDemand History for alternatives with less than or equal to %s movers in 1998:\n" % criterion) i_str = [str(x) for x in range(i)] info_file.write("outer_iteration, (movers)," + ",".join(i_str) + "\n") #info_file.write(", movers,\n") for row in com_rows_index: line = [str(x) for x in demand_history[row,]] info_file.write(str(alt_id[row]) + "," + ",".join(line) + "\n") #import pdb; pdb.set_trace() #export prob correlation history correlation_indices, prob_correlation = self.compute_prob_correlation(probability_old, probability, prob_hat, index, resources) info_file.write("\nCorrelation of Probabilities:\n") c_name = ['corr(p_ij p~_ij)', 'corr(p_ij p^_ij)', 'corr(p_ij dummy)', 'corr(p~_ij p^_ij)', 'corr(p~_ij dummy)', 'corr(p^_ij dummy)'] info_file.write("com_id, " + ",".join(c_name) + "\n") #info_file.write(", ,\n") for row in range(correlation_indices.size): line = [str(x) for x in prob_correlation[row,]] info_file.write(str(alt_id[correlation_indices[row]]) + "," + ",".join(line) + "\n") info_file.close() result['pi'] = pi return result logger.end_block() try:info_file.close() except:pass raise RuntimeError, "max iteration reached without convergence."
class AgentRelocationModel(Model): """Chooses agents for relocation (according to probabilities computed by the probabilities class). It includes all jobs that are unplaced. If probabilities is set to None, only unplaced agents are chosen. The run method returns indices of the chosen agents. """ def __init__(self, probabilities = "urbansim.rate_based_probabilities", choices = "opus_core.random_choices", location_id_name="grid_id", model_name = "Agent Relocation Model", debuglevel=0, resources=None ): self.model_name = model_name self.location_id_name = location_id_name self.debug = DebugPrinter(debuglevel) self.upc_sequence = None if probabilities is not None: self.upc_sequence = UPCFactory().get_model(utilities=None, probabilities=probabilities, choices=choices, debuglevel=debuglevel) self.resources = merge_resources_if_not_None(resources) def run(self, agent_set, resources=None, reset_attribute_value={}, append_unplaced_agents_index=True): self.resources.merge(resources) if agent_set.size()<=0: agent_set.get_id_attribute() if agent_set.size()<= 0: self.debug.print_debug("Nothing to be done.",2) return array([], dtype='int32') if self.upc_sequence and (self.upc_sequence.probability_class.rate_set or self.resources.get('relocation_rate', None)): self.resources.merge({agent_set.get_dataset_name():agent_set}) #to be compatible with old-style one-relocation_probabilities-module-per-model self.resources.merge({'agent_set':agent_set}) choices = self.upc_sequence.run(resources=self.resources) # choices have value 1 for agents that should be relocated, otherwise 0. movers_indices = where(choices>0)[0] else: movers_indices = array([], dtype='int32') if reset_attribute_value and movers_indices.size > 0: for key, value in reset_attribute_value.items(): agent_set.modify_attribute(name=key, data=resize(asarray(value), movers_indices.size), index=movers_indices) if append_unplaced_agents_index: # add unplaced agents unplaced_agents = where(agent_set.get_attribute(self.location_id_name) <= 0)[0] movers_indices = unique(concatenate((movers_indices, unplaced_agents))) logger.log_status("Number of movers: " + str(movers_indices.size)) return movers_indices def prepare_for_run(self, what=None, rate_dataset_name=None, rate_storage=None, rate_table=None, sample_rates=False, n=100, multiplicator=1, flush_rates=True): """ what - unused, argument kept to be compatible with old code """ from opus_core.datasets.dataset_factory import DatasetFactory from opus_core.session_configuration import SessionConfiguration if (rate_storage is None) or ((rate_table is None) and (rate_dataset_name is None)): return self.resources if not rate_dataset_name: rate_dataset_name = DatasetFactory().dataset_name_for_table(rate_table) rates = DatasetFactory().search_for_dataset(rate_dataset_name, package_order=SessionConfiguration().package_order, arguments={'in_storage':rate_storage, 'in_table_name':rate_table, } ) if sample_rates: cache_storage=None if flush_rates: cache_storage=rate_storage rates.sample_rates(n=n, cache_storage=cache_storage, multiplicator=multiplicator) self.resources.merge({rate_dataset_name:rates}) #to be compatible with old-style one-relocation_probabilities-module-per-model self.resources.merge({'relocation_rate':rates}) return self.resources ### In order to remove a circular dependency between this file and ### household_location_choice_model_creator, these unit tests were moved into ### urbansim.tests.test_agent_relocation_model.
class AgentRelocationModel(Model): """Chooses agents for relocation (according to probabilities computed by the probabilities class). It includes all jobs that are unplaced. If probabilities is set to None, only unplaced agents are chosen. The run method returns indices of the chosen agents. """ def __init__(self, probabilities="urbansim.rate_based_probabilities", choices="opus_core.random_choices", location_id_name="grid_id", model_name="Agent Relocation Model", debuglevel=0, resources=None): self.model_name = model_name self.location_id_name = location_id_name self.debug = DebugPrinter(debuglevel) self.upc_sequence = None if probabilities is not None: self.upc_sequence = UPCFactory().get_model( utilities=None, probabilities=probabilities, choices=choices, debuglevel=debuglevel) self.resources = merge_resources_if_not_None(resources) def run(self, agent_set, resources=None, reset_attribute_value={}, append_unplaced_agents_index=True): self.resources.merge(resources) if agent_set.size() <= 0: agent_set.get_id_attribute() if agent_set.size() <= 0: self.debug.print_debug("Nothing to be done.", 2) return array([], dtype='int32') if self.upc_sequence and (self.upc_sequence.probability_class.rate_set or self.resources.get( 'relocation_rate', None)): self.resources.merge( {agent_set.get_dataset_name(): agent_set} ) #to be compatible with old-style one-relocation_probabilities-module-per-model self.resources.merge({'agent_set': agent_set}) choices = self.upc_sequence.run(resources=self.resources) # choices have value 1 for agents that should be relocated, otherwise 0. movers_indices = where(choices > 0)[0] else: movers_indices = array([], dtype='int32') if reset_attribute_value and movers_indices.size > 0: for key, value in reset_attribute_value.items(): agent_set.modify_attribute(name=key, data=resize(asarray(value), movers_indices.size), index=movers_indices) if append_unplaced_agents_index: # add unplaced agents unplaced_agents = where( agent_set.get_attribute(self.location_id_name) <= 0)[0] movers_indices = unique( concatenate((movers_indices, unplaced_agents))) logger.log_status("Number of movers: " + str(movers_indices.size)) return movers_indices def prepare_for_run(self, what=None, rate_dataset_name=None, rate_storage=None, rate_table=None, sample_rates=False, n=100, multiplicator=1, flush_rates=True): """ what - unused, argument kept to be compatible with old code """ from opus_core.datasets.dataset_factory import DatasetFactory from opus_core.session_configuration import SessionConfiguration if (rate_storage is None) or ((rate_table is None) and (rate_dataset_name is None)): return self.resources if not rate_dataset_name: rate_dataset_name = DatasetFactory().dataset_name_for_table( rate_table) rates = DatasetFactory().search_for_dataset( rate_dataset_name, package_order=SessionConfiguration().package_order, arguments={ 'in_storage': rate_storage, 'in_table_name': rate_table, }) if sample_rates: cache_storage = None if flush_rates: cache_storage = rate_storage rates.sample_rates(n=n, cache_storage=cache_storage, multiplicator=multiplicator) self.resources.merge( {rate_dataset_name: rates} ) #to be compatible with old-style one-relocation_probabilities-module-per-model self.resources.merge({'relocation_rate': rates}) return self.resources ### In order to remove a circular dependency between this file and ### household_location_choice_model_creator, these unit tests were moved into ### urbansim.tests.test_agent_relocation_model.
def run(self, data, upc_sequence, resources=None): CLOSE = 0.01 self.mnl_probabilities=upc_sequence.probability_class self.bhhh_estimation = bhhh_mnl_estimation() modified_upc_sequence = UPCFactory().get_model( utilities=None, probabilities="opus_core.mnl_probabilities", choices=None) modified_upc_sequence.utility_class = upc_sequence.utility_class result = self.bhhh_estimation.run(data, modified_upc_sequence, resources) probability = modified_upc_sequence.get_probabilities() probability_0 = probability resources.check_obligatory_keys(["capacity"]) supply = resources["capacity"] if not isinstance(supply, ndarray): supply = array(supply) nsupply = supply.size max_iter = resources.get("max_iterations", None) if max_iter == None: max_iter = 100 # default index = resources.get("index", None) if index == None: index = arange(nsupply) neqs = probability.shape[1] nobs = probability.shape[0] if index.ndim <= 1: index = repeat(reshape(index, (1,index.shape[0])), nobs) resources.merge({"index":index}) # WARNING: THE SCALING OF DEMAND IS HARD CODED AND NEEDS TO BE MADE AN ARGUMENT # scale demand to represent 100% from a 0.2% sample demand = self.mnl_probabilities.get_demand(index, probability, nsupply)*50 #initial calculations sdratio = ma.filled(supply/ma.masked_where(demand==0, demand),2.0) sdratio = _round(sdratio, 1.0, atol=CLOSE) constrained_locations = logical_and(sdratio<1.0,demand-supply>CLOSE).astype("int8") unconstrained_locations = 1-constrained_locations excess_demand = (demand-supply)*constrained_locations global_excess_demand = excess_demand.sum() sdratio_matrix = sdratio[index] constrained_locations_matrix = constrained_locations[index] constrained_ex_ante = constrained_locations_matrix # Would like to include following print statements in debug printing # logger.log_status('Total demand:',demand.sum()) # logger.log_status('Total supply:',supply.sum()) logger.log_status('Global excess demand:',global_excess_demand) # logger.log_status('Constrained locations:',constrained_locations.sum()) unconstrained_locations_matrix = unconstrained_locations[index] # omega = ones(nobs,type=float32) # pi = self.constrain_probabilities.get_pi(sdratio_matrix, omega, constrained_locations_matrix, unconstrained_locations_matrix, nobs) omega = self.mnl_probabilities.get_omega(probability, constrained_locations_matrix, unconstrained_locations_matrix, sdratio_matrix) omega = _round(omega, 1.0, CLOSE) print 'Num of constrainted locations: ', constrained_locations.sum() print 'Num of unconstrainted locations: ', unconstrained_locations.sum() print 'Min Ex Ante Constraints:',min(constrained_ex_ante.sum(axis=1)) print 'Max Ex Ante Constraints:',max(constrained_ex_ante.sum(axis=1)) #print 'Omega shape',omega.shape #print 'Omega histogram',histogram(omega,0,4,40) print 'Minimum Omega',min(omega) print 'Maximum Omega',max(omega) print 'Mean Omega:',mean(omega) print 'Median Omega:',median(omega) print 'Sum Omega:',omega.sum() print 'Standard Deviation Omega:',standard_deviation(omega) print 'Count of Negative Omega',(where(omega<0,1,0).sum()) print 'Count of Omega < 1',(where(omega<1,1,0).sum()) print 'Count of Omega > 2',(where(omega>2,1,0).sum()) print 'Count of Omega > 4',(where(omega>4,1,0).sum()) average_omega = self.mnl_probabilities.get_average_omega(omega, probability, index, nsupply, nobs, demand) average_omega=_round(average_omega, 1.0, CLOSE) coef_names = resources.get("coefficient_names", None) if coef_names is not None: coef_names = array(coef_names.tolist()+["ln_pi"]) resources.merge({"coefficient_names":coef_names}) data=concatenate((data,ones((nobs,neqs,1),dtype=float32)), axis=2) prev_omega = omega prev_constrained_locations_matrix = constrained_locations_matrix for i in range(max_iter): print print 'Iteration',i pi = self.mnl_probabilities.get_pi(sdratio_matrix, omega, constrained_locations_matrix, unconstrained_locations_matrix, nobs) #print 'pi shape',pi.shape #print 'data shape', data.shape #print 'min_pi',min(pi,axis=1) #print 'max_pi',max(pi,axis=1) #print 'min_data',min(data,axis=1) #print 'max_data',max(data,axis=1) data[:,:,-1] = ln(pi) #data = concatenate((data,(pi[:,:,newaxis])),axis=-1) #print 'data shape after contatenating pi', data.shape result = self.bhhh_estimation.run(data, modified_upc_sequence, resources) #print #print 'result',result probability = modified_upc_sequence.get_probabilities() prob_hat = ma.filled(probability / pi, 0.0) # HARD CODED # scale new_demand from 0.2% to 100% demand_new = self.mnl_probabilities.get_demand(index, prob_hat, nsupply)*50 ##update supply-demand ratio sdratio = ma.filled(supply/ma.masked_where(demand_new==0, demand_new),2.0) sdratio = _round(sdratio, 1.0, CLOSE) sdratio_matrix = sdratio[index] constrained_locations = where(((average_omega*demand_new - supply) > CLOSE),1,0) unconstrained_locations = 1-constrained_locations constrained_locations_matrix = constrained_locations[index] unconstrained_locations_matrix = unconstrained_locations[index] constrained_ex_post = constrained_locations_matrix constrained_ex_post_not_ex_ante = where((constrained_ex_post - constrained_ex_ante)==1,1,0) constrained_ex_ante_not_ex_post = where((constrained_ex_post - constrained_ex_ante)==-1,1,0) #Assumption 5: if j belongs to constrained ex post and unconstrained ex ante, then p^i_j <= D_j / S_j print 'Number of individual violating Assumption 5: ', where((probability > 1 / sdratio_matrix)*constrained_ex_post_not_ex_ante)[0].size #Assumption 6: pi of constrained locations should be less than 1 print 'Number of individual violating Assumption 6: ', where((probability * constrained_ex_post).sum(axis=1) > (prob_hat * constrained_ex_post).sum(axis=1))[0].size ##OR ? #print 'Assumption 6: ', where(pi[where(constrained_locations_matrix)] > 1)[0].size print 'number of constrainted locations: ', constrained_locations.sum() print 'number of unconstrainted locations: ', unconstrained_locations.sum() print 'Min Ex Post Constraints:',min(constrained_ex_post.sum(axis=1)) print 'Max Ex Post Constraints:',max(constrained_ex_post.sum(axis=1)) print 'At Least 1 Constrained Ex Ante Not Ex Post*:',where(constrained_ex_ante_not_ex_post.sum(axis=1))[0].size print 'At Least 1 Constrained Ex Post Not Ex Ante:',where(constrained_ex_post_not_ex_ante.sum(axis=1))[0].size omega = self.mnl_probabilities.get_omega(prob_hat, constrained_locations_matrix, unconstrained_locations_matrix, sdratio_matrix) omega = _round(omega, 1.0, CLOSE) #print 'Omega histogram',histogram(omega,0,4,40) print 'Minimum Omega',min(omega) print 'Maximum Omega',max(omega) print 'Mean Omega:',mean(omega) print 'Median Omega:',median(omega) print 'Sum Omega:',omega.sum() print 'Standard Deviation Omega:',standard_deviation(omega) print 'Count of Negative Omega',(where(omega<0,1,0).sum()) print 'Count of Omega < 1: ',(where(omega<1,1,0).sum()) print 'Count of Omega > 2: ',(where(omega>2,1,0).sum()) print 'Count of Omega > 4: ',(where(omega>4,1,0).sum()) average_omega = self.mnl_probabilities.get_average_omega(omega, prob_hat, index, nsupply, nobs, demand_new) average_omega = _round(average_omega, 1.0, CLOSE) excess_demand = (demand_new-supply)*constrained_locations global_excess_demand = excess_demand.sum() #print 'Omega [i], [i-1]',prev_omega, omega, #print 'Constrained locations [i], [i-1]',constrained_locations_matrix, prev_constrained_locations_matrix print 'Global Excess Demand',global_excess_demand if ma.allclose(omega, prev_omega, atol=1e-3) or not any(constrained_locations_matrix - prev_constrained_ex_ante): print 'omega or constrained ex post unchanged: Convergence criterion achieved' break #if global_excess_demand < 1: #print 'Global excess demand < 1: Convergence criterion achieved' #break return result
def run(self, data, upc_sequence, resources=None): self.mnl_probabilities = upc_sequence.probability_class self.bhhh_estimation = bhhh_mnl_estimation() modified_upc_sequence = UPCFactory().get_model( utilities=None, probabilities="opus_core.mnl_probabilities", choices=None) modified_upc_sequence.utility_class = upc_sequence.utility_class N, neqs, V = data.shape max_iter = resources.get("max_iterations", 100) # default sc = SessionConfiguration() dataset_pool = sc.get_dataset_pool() sample_rate = dataset_pool.get_dataset("sample_rate") CLOSE = sc["CLOSE"] info_filename = sc["info_file"] info_filename = os.path.join('.', info_filename) info_file = open(info_filename, "a") constraint_dict = {1: 'constrained', 0: 'unconstrained'} swing_cases_fix = 0 #set swing alternatives to constrained (1) or unconstrained (0) prob_correlation = None choice_set = resources['_model_'].choice_set J = choice_set.size() alt_id = choice_set.get_id_attribute() movers = choice_set.get_attribute('movers') resources.check_obligatory_keys(["capacity_string"]) supply = choice_set.get_attribute(resources["capacity_string"]) index = resources.get("index", None) if index is None: # no sampling case, alternative set is the full choice_set index = arange(J) if index.ndim <= 1: index = repeat(index[newaxis, :], N, axis=0) if resources.get('aggregate_to_dataset', None): aggregate_dataset = dataset_pool.get_dataset( resources.get('aggregate_to_dataset')) choice_set_aggregate_id = choice_set.get_attribute( aggregate_dataset.get_id_name()[0]) index = aggregate_dataset.get_id_index( choice_set_aggregate_id[index].ravel()).reshape(index.shape) supply = aggregate_dataset.get_attribute( resources["capacity_string"]) J = aggregate_dataset.size() movers = aggregate_dataset.get_attribute("movers") demand_history = movers[:, newaxis] resources.merge({"index": index}) pi = ones(index.shape, dtype=float32) #initialize pi #average_omega = ones(J,dtype=float32) #initialize average_omega logger.start_block('Outer Loop') for i in range(max_iter): logger.log_status('Outer Loop Iteration %s' % i) result = self.bhhh_estimation.run(data, modified_upc_sequence, resources) del self.bhhh_estimation collect() self.bhhh_estimation = bhhh_mnl_estimation() probability = modified_upc_sequence.get_probabilities() if data.shape[2] == V: #insert a placeholder for ln(pi) in data data = concatenate((data, ones((N, neqs, 1), dtype=float32)), axis=2) coef_names = resources.get("coefficient_names") coef_names = concatenate((coef_names, array(["ln_pi"]))) resources.merge({"coefficient_names": coef_names}) else: beta_ln_pi = result['estimators'][where( coef_names == 'ln_pi')][0] logger.log_status("mu = 1/%s = %s" % (beta_ln_pi, 1 / beta_ln_pi)) prob_hat = safe_array_divide(probability, pi**beta_ln_pi) #prob_hat = safe_array_divide(probability, pi) prob_hat_sum = prob_hat.sum(axis=1, dtype=float32) if not ma.allclose(prob_hat_sum, 1.0): logger.log_status( "probability doesn't sum up to 1, with minimum %s, and maximum %s" % (prob_hat_sum.min(), prob_hat_sum.max())) probability = normalize(prob_hat) demand = self.mnl_probabilities.get_demand(index, probability, J) * 1 / sample_rate demand_history = concatenate((demand_history, demand[:, newaxis]), axis=1) sdratio = safe_array_divide( supply, demand, return_value_if_denominator_is_zero=2.0) sdratio_matrix = sdratio[index] ## debug info from numpy import histogram from opus_core.misc import unique cc = histogram(index.ravel(), unique(index.ravel()))[0] logger.log_status( "=================================================================" ) logger.log_status("Probability min: %s, max: %s" % (probability.min(), probability.max())) logger.log_status("Demand min: %s, max: %s" % (demand.min(), demand.max())) logger.log_status("sdratio min: %s, max: %s" % (sdratio.min(), sdratio.max())) logger.log_status("demand[sdratio==sdratio.min()]=%s" % demand[sdratio == sdratio.min()]) logger.log_status("demand[sdratio==sdratio.max()]=%s" % demand[sdratio == sdratio.max()]) logger.log_status( "Counts of unique submarkets in alternatives min: %s, max: %s" % (cc.min(), cc.max())) logger.log_status( "=================================================================" ) constrained_locations_matrix, omega, info = self.inner_loop( supply, demand, probability, index, sdratio_matrix, J, max_iteration=max_iter) inner_iterations, constrained_locations_history, swing_index, average_omega_history = info for idx in swing_index: logger.log_status( "swinging alt with id %s set to %s" % (alt_id[idx], constraint_dict[swing_cases_fix])) constrained_locations_matrix[index == idx] = swing_cases_fix if swing_index.size > 0: info_file.write("swing of constraints found with id %s \n" % alt_id[swing_index]) info_file.write("outer_iteration, %i, " % i + ", ".join([str(i)] * (len(inner_iterations))) + "\n") info_file.write("inner_iteration, , " + ", ".join(inner_iterations) + "\n") info_file.write("id, sdratio, " + ", ".join(["avg_omega"] * len(inner_iterations)) + "\n") for idx in swing_index: line = str(alt_id[idx]) + ',' line += str(sdratio[idx]) + ',' line += ",".join( [str(x) for x in average_omega_history[idx, ]]) line += "\n" info_file.write(line) info_file.write("\n") info_file.flush() outer_iterations = [str(i)] * len(inner_iterations) prob_min = [str(probability.min())] * len(inner_iterations) prob_max = [str(probability.max())] * len(inner_iterations) pi_new = self.mnl_probabilities.get_pi( sdratio_matrix, omega, constrained_locations_matrix) data[:, :, -1] = ln(pi_new) #diagnostic output if not ma.allclose(pi, pi_new, atol=CLOSE): if i > 0: #don't print this for the first iteration logger.log_status("min of abs(pi(l+1) - pi(l)): %s" % absolute(pi_new - pi).min()) logger.log_status("max of abs(pi(l+1) - pi(l)): %s" % absolute(pi_new - pi).max()) logger.log_status("mean of pi(l+1) - pi(l): %s" % (pi_new - pi).mean()) logger.log_status( 'Standard Deviation pi(l+1) - pi(l): %s' % standard_deviation(pi_new - pi)) logger.log_status('correlation of pi(l+1) and pi(l): %s' % corr(pi_new.ravel(), pi.ravel())[0, 1]) pi = pi_new probability_old = probability # keep probability of the previous loop, for statistics computation only else: #convergence criterion achieved, quiting outer loop logger.log_status( "pi(l) == pi(l+1): Convergence criterion achieved") info_file.write("\nConstrained Locations History:\n") info_file.write("outer_iteration," + ",".join(outer_iterations) + "\n") info_file.write("inner_iteration," + ",".join(inner_iterations) + "\n") info_file.write("minimum_probability," + ",".join(prob_min) + "\n") info_file.write("maximum_probability," + ",".join(prob_max) + "\n") for row in range(J): line = [ str(x) for x in constrained_locations_history[row, ] ] info_file.write( str(alt_id[row]) + "," + ",".join(line) + "\n") info_file.flush() info_file.write("\nDemand History:\n") i_str = [str(x) for x in range(i)] info_file.write("outer_iteration, (movers)," + ",".join(i_str) + "\n") #info_file.write(", ,\n") for row in range(J): line = [str(x) for x in demand_history[row, ]] info_file.write( str(alt_id[row]) + "," + ",".join(line) + "\n") demand_history_info_criteria = [500, 100, 50, 20] for criterion in demand_history_info_criteria: com_rows_index = where(movers <= criterion)[0] info_file.write( "\nDemand History for alternatives with less than or equal to %s movers in 1998:\n" % criterion) i_str = [str(x) for x in range(i)] info_file.write("outer_iteration, (movers)," + ",".join(i_str) + "\n") #info_file.write(", movers,\n") for row in com_rows_index: line = [str(x) for x in demand_history[row, ]] info_file.write( str(alt_id[row]) + "," + ",".join(line) + "\n") #import pdb; pdb.set_trace() #export prob correlation history correlation_indices, prob_correlation = self.compute_prob_correlation( probability_old, probability, prob_hat, index, resources) info_file.write("\nCorrelation of Probabilities:\n") c_name = [ 'corr(p_ij p~_ij)', 'corr(p_ij p^_ij)', 'corr(p_ij dummy)', 'corr(p~_ij p^_ij)', 'corr(p~_ij dummy)', 'corr(p^_ij dummy)' ] info_file.write("com_id, " + ",".join(c_name) + "\n") #info_file.write(", ,\n") for row in range(correlation_indices.size): line = [str(x) for x in prob_correlation[row, ]] info_file.write( str(alt_id[correlation_indices[row]]) + "," + ",".join(line) + "\n") info_file.close() result['pi'] = pi return result logger.end_block() try: info_file.close() except: pass raise RuntimeError, "max iteration reached without convergence."