def plot_coefficients(self, submodel=-2, exclude_constant=True, eqidx=0, plot=True, store_values_to_file=None): """ Plot a barchart of coefficient values. This can be used in a regression model, when coefficients are standardized (i.e. using the estimation module opus_core.estimate_linear_regression_standardized). store_values_to_file can be a file name where the values are stored. """ coef = self.get_coefficients(submodel) values = coef.get_coefficient_values() names = coef.get_coefficient_names() sd = coef.get_standard_errors() idx=ones(names.shape[1], dtype="bool") if exclude_constant: pos = coef.get_constants_positions() if pos.size > 0: idx[pos]=0 if store_values_to_file is not None: n = idx.sum() result = concatenate((reshape(names[eqidx, idx], (n,1)), reshape(values[eqidx, idx], (n,1)), reshape(sd[eqidx, idx], (n,1))), axis=1) write_to_text_file(store_values_to_file, array(['coefficient_name', 'estimate', 'standard_error']), delimiter='\t') write_table_to_text_file(store_values_to_file, result, delimiter='\t', mode='a') if plot: plot_barchart(values[eqidx, idx], labels = names[eqidx, idx], errors=sd[eqidx, idx]) else: return {'names': names[eqidx, idx], 'values': values[eqidx, idx], 'errors': sd[eqidx, idx]}
def export_weights_posterior_mean_and_variance( self, years, quantity_of_interest, directory, filename=None, use_bias_and_variance_from=None, ids=None): for year in years: self.set_posterior(year, quantity_of_interest, use_bias_and_variance_from) if filename is None: filename = quantity_of_interest file = os.path.join(directory, str(year) + '_' + filename) write_to_text_file(file, concatenate((array([0.]), self.get_weights())), delimiter=' ') write_to_text_file(file, concatenate( (array([0.]), self.get_posterior_component_variance())), mode='a', delimiter=' ') variable_list = self.get_variable_names() quantity_index = variable_list.index(quantity_of_interest) if ids is None: ids = self.m_ids means = zeros((ids.size, self.number_of_runs + 1)) means[:, 0] = ids means[self.observed_data.get_quantity_objects( )[quantity_index].get_dataset().get_id_index(ids), 1:means.shape[1]] = self.get_posterior_component_mean() write_table_to_text_file(file, means, mode='a')
def _do_export_values(self, variable, values, directory, prefix='', variable_prefix=''): filename = os.path.join(directory, "%s%s.%s%s" % (prefix, VariableName(variable).get_dataset_name(), variable_prefix, VariableName(variable).get_alias())) if self.ids_matching_values_from_mr[variable].ndim < values.ndim: data = concatenate((self.ids_matching_values_from_mr[variable][:,newaxis], values), axis=1) else: data = concatenate((self.ids_matching_values_from_mr[variable], values), axis=1) write_table_to_text_file(filename, data)
def write_posterior_mean_and_variance(self, mean_filename=None, variance_filename=None): if mean_filename is not None: write_table_to_text_file(mean_filename, self.get_posterior_component_mean()) if variance_filename is not None: write_to_text_file(variance_filename, self.get_posterior_component_variance(), delimiter=' ')
def export_weights_posterior_mean_and_variance(self, years, quantity_of_interest, directory, filename=None, use_bias_and_variance_from=None, ids = None, **kwargs): for year in years: self.set_posterior(year, quantity_of_interest, use_bias_and_variance_from, **kwargs) if filename is None: filename = quantity_of_interest file = os.path.join(directory, str(year) + '_' + filename) write_to_text_file(file, concatenate((array([0.]), self.get_weights())), delimiter=' ') write_to_text_file(file, concatenate((array([0.]), self.get_posterior_component_variance())), mode='a', delimiter=' ') variable_list = self.get_variable_names() quantity_index = variable_list.index(quantity_of_interest) if ids is None: ids = self.m_ids means = zeros((ids.size, self.number_of_runs+1)) means[:,0] = ids means[:,1:means.shape[1]] = self.get_posterior_component_mean() write_table_to_text_file(file, means, mode='a')
def _do_export_values(self, variable, values, directory, prefix='', variable_prefix=''): filename = os.path.join( directory, "%s%s.%s%s" % (prefix, VariableName(variable).get_dataset_name(), variable_prefix, VariableName(variable).get_alias())) if self.ids_matching_values_from_mr[variable].ndim < values.ndim: data = concatenate( (self.ids_matching_values_from_mr[variable][:, newaxis], values), axis=1) else: data = concatenate( (self.ids_matching_values_from_mr[variable], values), axis=1) write_table_to_text_file(filename, data)
def test_get_node_data_into_node_travel_data_set(self): temp_dir = tempfile.mkdtemp(prefix='opus_tmp') file1 = 'report1' temp_file1 = os.path.join(temp_dir, file1) write_to_text_file(temp_file1, array(['inode', 'jnode', 'timau', '@corr', 'len', 'result']), delimiter=' ') write_table_to_text_file(temp_file1, array([[1,2, 35.6, 4, 1.2, 0], [2,1, 23.5, 3, 0.3,100], [4,10, 2.1, 3, 0.5, 10], [3,1, 15.8, 4, 1.1, 5] ]), delimiter = ' ', mode='a') file2 = 'report2' temp_file2 = os.path.join(temp_dir, file2) write_to_text_file(temp_file2, array(['inode', 'jnode', 'volau', 'result']), delimiter=' ') write_table_to_text_file(temp_file2, array([[1,2, 110, 0], [3,1, 350, 400], [5,4, 200, 200]]), delimiter = ' ', mode='a') node_matrix_attribute_map = {file1: { 'timau':'travel_time', 'len':'distance', '@corr': 'corridor' }, file2: { 'volau': 'travel_volume' } } tm_output = TravelModelOutput() node_travel_data_set = tm_output.get_node_travel_data_set(node_matrix_attribute_map, temp_dir) # size should be 5, since there are 5 unique combinations of from_node, to_node self.assertEqual(node_travel_data_set.size(), 5) # the dataset should have 6 attributes self.assertEqual(len(node_travel_data_set.get_known_attribute_names()), 6) self.assertEqual('travel_time' in node_travel_data_set.get_known_attribute_names(), True) self.assertEqual('distance' in node_travel_data_set.get_known_attribute_names(), True) self.assertEqual('corridor' in node_travel_data_set.get_known_attribute_names(), True) self.assertEqual('travel_volume' in node_travel_data_set.get_known_attribute_names(), True) self.assertEqual('from_node_id' in node_travel_data_set.get_known_attribute_names(), True) self.assertEqual('to_node_id' in node_travel_data_set.get_known_attribute_names(), True) # check values of one node node = node_travel_data_set.get_data_element_by_id((3,1)) self.assertEqual(node.corridor, 4) self.assertEqual(node.travel_volume, 350) shutil.rmtree(temp_dir)
def run(self, data, coefficients, resources=None): """ Like linear_utilities, but in addition it runs linear utilities for modified data and stores utilities when each variable is set to its 5%, 95% quantiles, keeping the other variables at their median. Last row in the resulting file is the difference in utilities between these two. The file name can be passed in resources - entry 'utilities_diagnose_file'. """ if data.ndim < 3: raise StandardError, "Argument 'data' must be a 3D numpy array." if not isinstance(resources, Resources): resources= Resources(resources) nobs, neqs, nvar = data.shape medians = zeros(nvar, dtype=float32) quant = zeros((2,nvar), dtype=float32) data_with_medians = array(data[0,:,:]) for ivar in range(nvar): # compute medain and quantiles for each variable medians[ivar], quant[0,ivar], quant[1,ivar] = quantile(data[:,:,ivar].ravel(), array([0.5, 0.05, 0.95])) data_with_medians[:,ivar] = medians[ivar] file_name = resources.get("utilities_diagnose_file", "util") if resources.get("submodel", None) is not None: file_name = "%s_submodel_%s" % (file_name, resources.get("submodel", 1)) diagnose_utilities = zeros((3, nvar), dtype=float32) argcor = () for ivar in range(nvar): # iterate over variables for iquant in [0,1]: # 0 for 5% quantile, 1 for 95% quantile mod_data = array(data_with_medians).reshape(1,neqs, nvar) # copy original data mod_data[0,:,ivar] = quant[iquant, ivar] utility = linear_utilities.run(self, mod_data, coefficients, resources) diagnose_utilities[iquant, ivar] = utility[0,0] argcor = argcor + (data[:,:,ivar].ravel(),) diagnose_utilities[2,:] = diagnose_utilities[1,:] - diagnose_utilities[0,:] coef_names = resources.get("coefficient_names", map(lambda x: 'x%s' % x, arange(nvar)+1)) write_to_text_file(file_name, coef_names, delimiter=' ') write_table_to_text_file( file_name, diagnose_utilities, mode='ab') logger.log_status("Diagnosed utilities written into %s." % file_name) return linear_utilities.run(self, data, coefficients, resources)
def export_confidence_intervals(self, confidence_levels, filename, delimiter='\t'): """Export confidence intervals into a file. confidence_levels is a list of desired confidence levels. The resulting file has a id column, 'mean' column and for each level its lower bound and upper bound columns. The method generate_posterior_distribution should be called prior to this method. """ if not isinstance(confidence_levels, list): confidence_levels = [confidence_levels] lcl = len(confidence_levels) result = zeros((self.simulated_values_ids.size, 2+2*lcl), dtype='float32') result[:,0] = self.simulated_values_ids result[:,1] = self.get_quantity_from_simulated_values("mean") clheader = [] for cl in confidence_levels: clheader = clheader + ['lower_%s' % cl, 'upper_%s' % cl] write_to_text_file(filename, ['id', 'mean'] + clheader, delimiter=delimiter) i = 2 for cl in confidence_levels: ci = self.get_probability_interval(cl/100.0) result[:,i:(i+2)] = transpose(ci) i = i+2 write_table_to_text_file(filename, result, mode='a', delimiter=delimiter)
def write_into_run_id_file(self): result = array( map(lambda (x, y): [x, y[0], y[1]], self.run_ids_dict.iteritems())) write_table_to_text_file(self.run_id_file, result)
def write_values_from_multiple_runs(self, filename, transformed_back=True): write_table_to_text_file(filename, self.get_predicted_values(transformed_back=transformed_back))
def write_expected_values(self, filename, index, transformed_back=True): write_table_to_text_file(filename, self.get_expected_values_by_index(index, transformed_back))
def write_simulated_values(self, filename): write_table_to_text_file(filename, self.simulated_values)
def test_get_node_data_into_node_travel_data_set(self): temp_dir = tempfile.mkdtemp(prefix='opus_tmp') file1 = 'report1' temp_file1 = os.path.join(temp_dir, file1) write_to_text_file( temp_file1, array(['inode', 'jnode', 'timau', '@corr', 'len', 'result']), delimiter=' ') write_table_to_text_file(temp_file1, array([[1, 2, 35.6, 4, 1.2, 0], [2, 1, 23.5, 3, 0.3, 100], [4, 10, 2.1, 3, 0.5, 10], [3, 1, 15.8, 4, 1.1, 5]]), delimiter=' ', mode='a') file2 = 'report2' temp_file2 = os.path.join(temp_dir, file2) write_to_text_file(temp_file2, array(['inode', 'jnode', 'volau', 'result']), delimiter=' ') write_table_to_text_file(temp_file2, array([[1, 2, 110, 0], [3, 1, 350, 400], [5, 4, 200, 200]]), delimiter=' ', mode='a') node_matrix_attribute_map = { file1: { 'timau': 'travel_time', 'len': 'distance', '@corr': 'corridor' }, file2: { 'volau': 'travel_volume' } } tm_output = TravelModelOutput() node_travel_data_set = tm_output.get_node_travel_data_set( node_matrix_attribute_map, temp_dir) # size should be 5, since there are 5 unique combinations of from_node, to_node self.assertEqual(node_travel_data_set.size(), 5) # the dataset should have 6 attributes self.assertEqual(len(node_travel_data_set.get_known_attribute_names()), 6) self.assertEqual( 'travel_time' in node_travel_data_set.get_known_attribute_names(), True) self.assertEqual( 'distance' in node_travel_data_set.get_known_attribute_names(), True) self.assertEqual( 'corridor' in node_travel_data_set.get_known_attribute_names(), True) self.assertEqual( 'travel_volume' in node_travel_data_set.get_known_attribute_names(), True) self.assertEqual( 'from_node_id' in node_travel_data_set.get_known_attribute_names(), True) self.assertEqual( 'to_node_id' in node_travel_data_set.get_known_attribute_names(), True) # check values of one node node = node_travel_data_set.get_data_element_by_id((3, 1)) self.assertEqual(node.corridor, 4) self.assertEqual(node.travel_volume, 350) shutil.rmtree(temp_dir)
def export_quantiles(bm, outdir, years=[2010, 2040], repl=10000, validation_year=2010, validation_geography='faz', propfac_hh=0.95, propfac_jobs=3.5, propfac_pop=3.9, no_propagation=[False], aggregate_to=None, store_simulated=False, **kwargs): header_base = ['mean', 'median', 'lower_50', 'upper_50', 'lower_80', 'upper_80', 'lower_90', 'upper_90', 'lower_95', 'upper_95'] idcolname = {'zone': 'zone_id', 'faz': 'faz_id', 'large_area': 'large_area_id', 'city': 'city_id', 'tract10': 'tract10_id', 'reggeo': 'reggeo_id'} header = {} for g in idcolname.keys(): header[g] = array([idcolname[g]] + header_base)[newaxis,:] if aggregate_to is not None: header[validation_geography] = header[aggregate_to] vars = {'household':hhs_vars, 'job':jobs_vars, 'population':pop_vars} for year in years: if year < validation_year: continue #for nopropag in [True, False]: for nopropag in no_propagation: if not nopropag and year == validation_year: continue #for transform in [True, False]: transform = True propfac = {} for additive_prop in [True, False]: #for additive_prop in [True]: if nopropag and not additive_prop: continue if additive_prop: if nopropag: propfac['household'] = 0 propfac['job'] = 0 propfac['population'] = 0 else: propfac['household'] = propfac_hh propfac['job'] = propfac_jobs propfac['population'] = propfac_pop else: propfac['household'] = 1 propfac['job'] = 1 propfac['population'] = 1 if not nopropag: suffix = "_propf" else: suffix = "" if not transform: suffix = suffix + "_raw" if additive_prop and not nopropag: suffix = suffix + "_add" seed(1) #for bias in [False, True]: for bias in [False]: if bias: suffix = suffix + "_bias" addpropbias = not bias #bmf = deepcopy(bm) for indicator in vars.keys(): if aggregate_to is None: exact = True # calibration geography is the same as simulation target bm.set_posterior(year=year, quantity_of_interest=vars[indicator][validation_geography], additive_propagation=[addpropbias, additive_prop], propagation_factor=[0, propfac[indicator]]) mean_hhs = bm.get_posterior_component_mean().mean(axis=1) median = bm.get_exact_quantile(0.5, transformed_back=transform) hhids = bm.get_m_ids() if transform: mean_hhs = mean_hhs**2 file_geo = validation_geography else: # aggregation exact = False posterior_hhs = bm.generate_posterior_distribution(year=year, quantity_of_interest=vars[indicator][validation_geography], aggregate_to=aggregate_to, replicates=repl, omit_bias=not bias, no_propagation=nopropag, additive_propagation=[addpropbias, additive_prop], propagation_factor=[0, propfac[indicator]], transformed_back=transform, **kwargs) mean_hhs = bm.get_quantity_from_simulated_values("mean") median = bm.get_quantity_from_simulated_values("median") hhids = bm.simulated_values_ids file_geo = aggregate_to if store_simulated: bm.write_simulated_values(os.path.join(outdir, "%s_simulated_values_%s" % (aggregate_to, indicator))) #write_table_to_text_file(os.path.join(outdir, '%s_ids_%s' % (aggregate_to, indicator)), bm.simulated_values_ids[:,newaxis], delimiter='\t') prob80_hhs = bm.get_probability_interval(80, exact = exact, transformed_back=transform) prob50_hhs = bm.get_probability_interval(50, exact = exact, transformed_back=transform) prob90_hhs = bm.get_probability_interval(90, exact = exact, transformed_back=transform) prob95_hhs = bm.get_probability_interval(95, exact = exact, transformed_back=transform) write_table_to_text_file(os.path.join(outdir, '%s_%s_ci_%s%s' % (file_geo, indicator, year, suffix)), header[validation_geography]) write_table_to_text_file(os.path.join(outdir, '%s_%s_ci_%s%s' % (file_geo, indicator, year, suffix)), round(concatenate((hhids[:,newaxis], mean_hhs[:,newaxis], median[:,newaxis], prob50_hhs, prob80_hhs, prob90_hhs, prob95_hhs), axis=1)).astype('int32'), delimiter='\t', mode='a')
def write_weight_components(self, filename): l = len(self.get_weight_components().keys()) weight_matrix = zeros((l, self.number_of_runs)) for i in range(l): weight_matrix[i,:] = self.get_weight_components()[i] write_table_to_text_file(filename, weight_matrix)
def write_into_run_id_file(self): result = array(map(lambda(x,y): [x,y[0], y[1]], self.run_ids_dict.iteritems())) write_table_to_text_file(self.run_id_file, result)
def match_parcels_to_constraints_and_templates(parcel_dataset, development_template_dataset, output_dir, log_scale=True, strict=True, output_points=False, parcel_index=None, template_index=None, consider_constraints_as_rules=True, template_opus_path="urbansim_parcel.development_template", dataset_pool=None, resources=None): """ This function matches parcels to their constraints and templates and gives a summary about how many parcels have no match. It also creates a plot for each GLU and unit type of template ranges and densities. parcel_index - 1D array, indices of parcel_dataset (default is all parcels). template_index - index to templates that are available (default is all templates). If strict is True, parcels without templates are considered across GLU, otherwise only within each GLU. """ if not os.path.exists(output_dir): os.makedirs(output_dir) resources = Resources(resources) debug = resources.get("debug", 0) if not isinstance(debug, DebugPrinter): debug = DebugPrinter(debug) if parcel_index is not None and parcel_index.size <= 0: return None if parcel_index is not None: index1 = parcel_index else: index1 = arange(parcel_dataset.size()) if template_index is not None: index2 = template_index else: index2 = arange(development_template_dataset.size()) has_constraint_dataset = True try: constraints = dataset_pool.get_dataset("development_constraint") constraints.load_dataset_if_not_loaded() except: has_constraint_dataset = False parcels_glu = parcel_dataset.compute_variables(['parcel.disaggregate(land_use_type.generic_land_use_type_id)'], dataset_pool=dataset_pool) if has_constraint_dataset: constraint_types = unique(constraints.get_attribute("constraint_type")) #unit_per_acre, far etc development_template_dataset.compute_variables(map(lambda x: "%s.%s" % (template_opus_path, x), constraint_types), dataset_pool) parcel_dataset.get_development_constraints(constraints, dataset_pool, index=index1, consider_constraints_as_rules=consider_constraints_as_rules) generic_land_use_type_ids = development_template_dataset.compute_variables("urbansim_parcel.development_template.generic_land_use_type_id", dataset_pool=dataset_pool) parcel_ids = parcel_dataset.get_id_attribute() template_ids = development_template_dataset.get_id_attribute() has_template = zeros(index1.size, dtype="int32") vacant_land = parcel_dataset.compute_variables(['urbansim_parcel.parcel.vacant_land_area'], dataset_pool=dataset_pool)[index1] is_vacant = vacant_land>0 #vacant_land = vacant_land*logical_or(parcels_glu==1, parcels_glu==2) is_developable_parcel = zeros(index1.size, dtype="int32") accepted_by_constraints = zeros(index1.size, dtype="int32") #parcels_to_template = {} parcels_to_template_acc_by_constr = {} density_types = development_template_dataset['density_type'] parcels_acc_by_constr_wo_templ = {} parcels_acc_by_constr = {} #pidx = parcel_dataset.get_id_index(804461) logger.start_block("Combine parcels, templates and constraints") for i_template in index2: this_template_id = template_ids[i_template] fit_indicator = ones(index1.size, dtype="bool8") parcels_to_template_acc_by_constr[this_template_id] = [] this_templ_accepted_by_constraints = zeros(index1.size, dtype="int32") has_this_template = zeros(index1.size, dtype="int32") if has_constraint_dataset: generic_land_use_type_id = generic_land_use_type_ids[i_template] if generic_land_use_type_id not in parcels_acc_by_constr_wo_templ.keys(): parcels_acc_by_constr_wo_templ[generic_land_use_type_id] = zeros(index1.size, dtype="int32") if generic_land_use_type_id not in parcels_acc_by_constr.keys(): parcels_acc_by_constr[generic_land_use_type_id] = zeros(index1.size, dtype="int32") #if generic_land_use_type_id not in [1,2]: # continue units_proposed = parcel_dataset.compute_variables(['psrc_parcel.parcel.units_proposed_for_template_%s' % this_template_id], dataset_pool=dataset_pool)[index1] is_size_fit = parcel_dataset.compute_variables(['psrc_parcel.parcel.is_size_fit_for_template_%s' % this_template_id], dataset_pool=dataset_pool)[index1] for constraint_type, constraint in parcel_dataset.development_constraints[generic_land_use_type_id].iteritems(): if density_types[i_template] <> constraint_type: continue template_attribute = development_template_dataset.get_attribute(constraint_type)[i_template] #density converted to constraint variable name if template_attribute == 0: continue min_constraint = constraint[:, 0].copy() max_constraint = constraint[:, 1].copy() ## treat -1 as unconstrained w_unconstr = min_constraint == -1 if w_unconstr.any(): min_constraint[w_unconstr] = template_attribute w_unconstr = max_constraint == -1 if w_unconstr.any(): max_constraint[w_unconstr] = template_attribute this_accepted_by_constraints = logical_and(template_attribute >= min_constraint, template_attribute <= max_constraint) fit_indicator = logical_and(fit_indicator, logical_and(logical_and(this_accepted_by_constraints, units_proposed > 0), is_size_fit)) is_developable_parcel = logical_or(is_developable_parcel, max_constraint > 0) this_templ_accepted_by_constraints = logical_or(this_templ_accepted_by_constraints, logical_and(is_developable_parcel, logical_and(this_accepted_by_constraints, units_proposed > 0))) has_this_template = logical_or(has_this_template, fit_indicator) accepted_by_constraints = logical_or(accepted_by_constraints, this_templ_accepted_by_constraints) has_template = logical_or(has_template, has_this_template) #parcels_to_template[this_template_id] = where(logical_and(vacant_land>0, # logical_and(logical_and(is_developable_parcel, this_accepted_by_constraints), # logical_not(fit_indicator))))[0] #parcels_to_template_acc_by_constr[this_template_id].append(where(accepted_by_constraints)[0].tolist()) not_accepted = logical_and(this_templ_accepted_by_constraints, logical_and(logical_not(has_this_template), is_vacant)) parcels_to_template_acc_by_constr[this_template_id].append(where(not_accepted)[0].tolist()) parcels_acc_by_constr_wo_templ[generic_land_use_type_id] = logical_or(parcels_acc_by_constr_wo_templ[generic_land_use_type_id], not_accepted) parcels_acc_by_constr[generic_land_use_type_id] = logical_or(parcels_acc_by_constr[generic_land_use_type_id], logical_and(this_templ_accepted_by_constraints, is_vacant)) #if fit_indicator[pidx]: # print 'Parcel 804461: template %s accepted.' % this_template_id logger.end_block() ### Print summary ################## unique_glu = parcels_acc_by_constr_wo_templ.keys() #parcels_wo_templ = zeros(index1.size, dtype="int32") #parcels_wo_templ = where(logical_and(vacant_land>0, logical_and(is_developable_parcel, logical_not(has_template))))[0] #nr_parcels_wo_templ = parcels_wo_templ.size #is_vacant = vacant_land>0 #logger.log_status("\nGLU\tvacant land\tconstraint out\tno template") logger.log_status("\nGLU\tconsidered\tno template") no_glu_templ = [] parcels_wo_temp_by_glu = {} sum1 = 0 sum2 = 0 parcels_wo_templ = logical_not(has_template) for glu in unique_glu: if strict: parcels_acc_by_constr_wo_templ[glu] = logical_and(parcels_acc_by_constr_wo_templ[glu], parcels_wo_templ) #if glu == 3: #parcels_wo_templ = logical_or(parcels_wo_templ, parcels_acc_by_constr_wo_templ[glu]) # if glu not in generic_land_use_type_ids: # no_glu_templ.append(glu) #idx = parcels_glu==glu # if idx.sum() > 0: # logger.log_status("%s\t%7i\t\t%7i\t\t%7i" % (glu, is_vacant[idx].sum(), # is_vacant[idx].sum() - logical_and(is_vacant[idx], is_developable_parcel[idx]).sum(), # logical_and(is_vacant[idx], logical_and(is_developable_parcel[idx], logical_not(has_template[idx]))).sum())) # parcels_wo_temp_by_glu[glu] = where(logical_and(idx, logical_and(is_vacant, # logical_and(is_developable_parcel, logical_not(has_template)))))[0] logger.log_status("%s\t%7i\t\t%7i" % (glu, parcels_acc_by_constr[glu].sum(), parcels_acc_by_constr_wo_templ[glu].sum())) sum1 = sum1 + parcels_acc_by_constr[glu].sum() sum2 = sum2 + parcels_acc_by_constr_wo_templ[glu].sum() logger.log_status("\nall\t%7i\t\t%7i" % (sum1, sum2)) #if len(no_glu_templ) > 0: # logger.log_status("\nNo templates for GLUs: %s" % no_glu_templ) ### Create plots ################# templ_min_max = {} for glu in unique_glu: gidx = where(parcels_acc_by_constr_wo_templ[glu])[0] logger.start_block("Creating figures for GLU %s using %s parcels" % (glu,gidx.size)) templ_min_max[glu] = [] max_land_sqft = {'far': 0, 'units_per_acre': 0} min_land_sqft = {'far': 9999999, 'units_per_acre': 9999999} max_templ_attr = {'far': 0, 'units_per_acre': 0} min_templ_attr = {'far': 999999, 'units_per_acre': 9999999} xy = {'far':[], 'units_per_acre':[]} points = {'far':zeros((0,3)), 'units_per_acre':zeros((0,3))} npoints = {'far': 0, 'units_per_acre': 0} for i_template in index2: if glu <> generic_land_use_type_ids[i_template]: continue this_template_id = template_ids[i_template] #units_proposed = parcel_dataset['units_proposed_for_template_%s' % this_template_id] #is_size_fit = parcel_dataset['is_size_fit_for_template_%s' % this_template_id] #is_constraint = zeros(parcel_dataset.size(), dtype='bool8') #is_constraint[array(parcels_to_template_acc_by_constr[this_template_id])]=True #is_size_fit = logical_and(logical_and(logical_not(is_size_fit), # logical_and(is_vacant, units_proposed>0)), # logical_and(is_constraint, # is_developable_parcel)) missed_to_match = zeros(parcel_dataset.size(), dtype='bool8') missed_to_match[(unique(array(parcels_to_template_acc_by_constr[this_template_id]).flatten())).astype('int32')] = True missed_to_match = where(logical_and(missed_to_match, parcels_acc_by_constr_wo_templ[glu]))[0] #missed_to_match = unique(array(parcels_to_template_acc_by_constr[this_template_id]).flatten()) for constraint_type, constraint in parcel_dataset.development_constraints[glu].iteritems(): if density_types[i_template] <> constraint_type: continue template_attribute = development_template_dataset.get_attribute(constraint_type)[i_template] #density converted to constraint variable name if template_attribute == 0: continue templ_min_max[glu].append([development_template_dataset["land_sqft_min"][i_template], development_template_dataset["land_sqft_max"][i_template]]) xy[constraint_type] = xy[constraint_type] + [[development_template_dataset["land_sqft_min"][i_template], development_template_dataset["land_sqft_max"][i_template]], [template_attribute, template_attribute]] #if is_size_fit[gidx].sum() > 0: if missed_to_match.size > 0: npoints[constraint_type] = npoints[constraint_type] + missed_to_match.size #is_size_fit[gidx].sum() #if is_size_fit[gidx].sum() > 100: if missed_to_match.size > 100: draw = sample_noreplace(missed_to_match, 100) thisidx = draw else: thisidx = missed_to_match points[constraint_type] = concatenate((points[constraint_type], concatenate((parcel_dataset['vacant_land_area'][thisidx][:,newaxis], template_attribute*ones((thisidx.size,1)), parcel_ids[thisidx][:,newaxis]), axis=1)), axis=0) max_land_sqft[constraint_type] = max(max_land_sqft[constraint_type], parcel_dataset['vacant_land_area'][thisidx].max()) min_land_sqft[constraint_type] = min(min_land_sqft[constraint_type], parcel_dataset['vacant_land_area'][thisidx].max()) max_templ_attr[constraint_type] = max(max_templ_attr[constraint_type], template_attribute) min_templ_attr[constraint_type] = min(min_templ_attr[constraint_type], template_attribute) import matplotlib.ticker as ticker import matplotlib.pyplot as plt def myexp(x, pos): return '%i' % (round(exp(x))) def myexp2(x, pos): return '%.2f' % (round(exp(x), 2)) for type in ['far', 'units_per_acre']: if points[type].size == 0: continue #print xy[type] lxy = array(xy[type]) dots = points[type][:,0:2] minx = min_land_sqft[type]-100 maxx = max_land_sqft[type]+100 miny = min_templ_attr[type]-0.05 maxy = max_templ_attr[type]+0.05 if log_scale: lxy = log(lxy) dots = log(dots) minx = log(minx) maxx = log(maxx) miny = log(miny) maxy = log(maxy) fig = plt.figure() ax = fig.add_subplot(111) lines = ax.plot(*lxy) # template lines po = ax.plot(dots[:,0], dots[:,1]) # parcel points if log_scale: xformatter = ticker.FuncFormatter(myexp) yformatter = ticker.FuncFormatter(myexp2) ax.xaxis.set_major_formatter(xformatter) ax.yaxis.set_major_formatter(yformatter) # The following would be better but throws an error #locator = ticker.LogLocator(base=2.718282, subs=0.1) #ax.xaxis.set_major_locator(locator) plt.setp(lines, color='b', linewidth=1) plt.setp(po, marker='o', linestyle='None', linewidth=0) ax.axis([min(dots[:,0].min(), minx), max(dots[:,0].max(), maxx), min(dots[:,1].min(), miny), max(dots[:,1].max(), maxy)]) plt.title('GLU: %s, units: %s, missing: %s' % (glu, type, npoints[type])) #ax.grid(True) plt.xlabel('land sqft range') plt.ylabel('density') log_suffix = '' if log_scale: log_suffix = '_log' plt.savefig(os.path.join(output_dir, 'match_templates%s_%s_%s.pdf' % (log_suffix, glu, type))) plt.close() #plt.show() if output_points: #if glu == 3: write_table_to_text_file(os.path.join(output_dir, 'points_%s_%s.txt' % (glu, type)), points[type], delimiter=', ') logger.end_block() logger.log_status('Resulting figures stored into %s' % output_dir) return parcel_ids[index1][parcels_wo_templ]