def prepare_for_estimate(self, specification_dict = None, specification_storage=None, specification_table=None, agent_set=None, agents_for_estimation_storage=None, agents_for_estimation_table=None, filter_for_estimation_set=None, data_objects=None): specification = get_specification_for_estimation(specification_dict, specification_storage, specification_table) if self.filter is not None: agents_index = where( self.proposal_set.compute_variables(self.filter) )[0] id_attribute_name = ['parcel_id', 'template_id', 'is_redevelopment'] if agents_for_estimation_storage is not None: estimation_set = Dataset(in_storage = agents_for_estimation_storage, in_table_name=agents_for_estimation_table, id_name=id_attribute_name, dataset_name=agent_set.get_dataset_name()) filter_index = arange(estimation_set.size()) if filter_for_estimation_set: filter_index = where(estimation_set.compute_variables(filter_for_estimation_set, resources=Resources(data_objects)))[0] estimation_set.subset_by_index(filter_index, flush_attributes_if_not_loaded=False) id_attributes = None for attr_name in id_attribute_name: attr_value = agent_set.get_attribute_as_column(attr_name) if id_attributes == None: id_attributes = attr_value else: id_attributes = concatenate((id_attributes, attr_value), axis=1) id_index = estimation_set.try_get_id_index(id_attributes, return_value_if_not_found=-1) status_id = 2 * ones(agent_set.size(), dtype="int8") status_id[where(id_index != -1)] = 1 name = self.choice_attribute_name.get_alias() if name in agent_set.get_known_attribute_names(): agent_set.set_values_of_one_attribute(name, status_id[where(id_index != -1)], where(id_index!=-1)[0]) else: agent_set.add_primary_attribute(status_id, name) return (specification, agents_index)
class Test(opus_unittest.OpusTestCase): def setUp(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table( table_name='households', table_data={ 'household_id': arange(10) + 1, # 'household_id':array([1, 2, 3, 4, 5, 6, 7, 8]), # 'income' :array([1, 3, 2, 1, 3, 8, 5, 4]), # #'category_id' :array([1, 2, 2, 1, 2, 3, 3, 2]), # 'building_id' :array([1, 2, 4, 3, 3, 2, 4, 2]), ##'large_area_id':array([1, 1, 2, 3, 3, 1, 2, 1]), # 'grid_id': arange(-1, 9, 1) + 1, 'lucky': array([1, 0, 1, 0, 1, 1, 1, 1, 0, 0]) }) storage.write_table( table_name='gridcells', table_data={ #'building_id': array([1, 2, 3, 4]), #'large_area_id': array([1, 1, 3, 2]), 'grid_id': arange(15) + 1, 'filter': array([0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1]), 'weight': array([0.1, 9, 15, 2, 5, 1, 6, 2.1, .3, 4, 3, 1, 10, 8, 7]) }) dataset_pool = SessionConfiguration( in_storage=storage).get_dataset_pool() #create households self.households = Dataset(in_storage=storage, in_table_name='households', id_name="household_id", dataset_name="household") # create gridcells self.gridcells = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name="gridcell") dataset_pool.replace_dataset('household', self.households) dataset_pool.replace_dataset('gridcell', self.gridcells) def test_1(self): """""" sample_size = 5 # check the individual gridcells # This is a stochastic model, so it may legitimately fail occassionally. index1 = where(self.households.get_attribute("lucky"))[0] #index2 = where(self.gridcells.get_attribute("filter"))[0] weight = self.gridcells.get_attribute("weight") estimation_config = { "agent_category_definition": ["household.lucky"], "choice_category_definition": ["gridcell.filter+1"] } for icc in [0, 1]: #include_chosen_choice? #icc = sample([0,1],1) sampler_ret = weighted_sampler_by_category().run( dataset1=self.households, dataset2=self.gridcells, index1=index1, sample_size=sample_size, include_chosen_choice=icc, resources=estimation_config) # get results sampled_index = sampler_ret.get_2d_index() chosen_choices = UNPLACED_ID * ones(index1.size, dtype=DTYPE) where_chosen = where(sampler_ret.get_attribute("chosen_choice")) chosen_choices[where_chosen[0]] = where_chosen[1] self.assertEqual(sampled_index.shape, (index1.size, sample_size)) if icc: placed_agents_index = self.gridcells.try_get_id_index( self.households.get_attribute("grid_id")[index1], UNPLACED_ID) chosen_choice_index = resize(array([UNPLACED_ID], dtype=DTYPE), index1.shape) w = where(chosen_choices >= 0)[0] # for 64 bit machines, need to coerce the type to int32 -- on a # 32 bit machine the astype(int32) doesn't do anything chosen_choice_index[w] = sampled_index[ w, chosen_choices[w]].astype(int32) self.assert_( alltrue(equal(placed_agents_index, chosen_choice_index))) sampled_index = sampled_index[:, 1:] self.assert_( alltrue( lookup(sampled_index.ravel(), arange(self.gridcells.size()), index_if_not_found=UNPLACED_ID) != UNPLACED_ID)) self.assert_(all(not_equal(weight[sampled_index], 0.0)))
class Test(opus_unittest.OpusTestCase): def setUp(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='households', table_data={ 'household_id': arange(10) + 1, 'grid_id': arange(-1, 9, 1) + 1, 'lucky': array([1, 0, 1, 0, 1, 1, 1, 1, 0, 0]) }) storage.write_table( table_name='gridcells', table_data={ 'grid_id': arange(15) + 1, 'filter': array([0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 1, 1]), 'weight': array([0.1, 9, 15, 2, 5, 1, 6, 2.1, .3, 4, 3, 1, 10, 8, 7]) }) #create households self.households = Dataset(in_storage=storage, in_table_name='households', id_name="household_id", dataset_name="household") # create gridcells self.gridcells = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name="gridcell") def test_1d_weight_array(self): """""" sample_size = 5 # check the individual gridcells # This is a stochastic model, so it may legitimately fail occassionally. index1 = where(self.households.get_attribute("lucky"))[0] index2 = where(self.gridcells.get_attribute("filter"))[0] weight = self.gridcells.get_attribute("weight") for icc in [0, 1]: #include_chosen_choice? #icc = sample([0,1],1) sampler_ret = weighted_sampler().run(dataset1=self.households, dataset2=self.gridcells, index1=index1, index2=index2, sample_size=sample_size, weight="weight", include_chosen_choice=icc) # get results sampled_index = sampler_ret.get_2d_index() chosen_choices = UNPLACED_ID * ones(index1.size, dtype=DTYPE) where_chosen = where(sampler_ret.get_attribute("chosen_choice")) chosen_choices[where_chosen[0]] = where_chosen[1] sample_results = sampled_index, chosen_choices sampled_index = sample_results[0] self.assertEqual(sampled_index.shape, (index1.size, sample_size)) if icc: placed_agents_index = self.gridcells.try_get_id_index( self.households.get_attribute("grid_id")[index1], UNPLACED_ID) chosen_choice_index = resize(array([UNPLACED_ID], dtype=DTYPE), index1.shape) w = where(chosen_choices >= 0)[0] # for 64 bit machines, need to coerce the type to int32 -- on a # 32 bit machine the astype(int32) doesn't do anything chosen_choice_index[w] = sampled_index[w, chosen_choices[w]] self.assert_( alltrue(equal(placed_agents_index, chosen_choice_index))) sampled_index = sampled_index[:, 1:] self.assert_( alltrue( lookup(sampled_index.ravel(), index2, index_if_not_found=UNPLACED_ID) != UNPLACED_ID)) self.assert_(all(not_equal(weight[sampled_index], 0.0))) def test_2d_weight_array(self): #2d weight sample_size = 5 n = self.households.size() index1 = where(self.households.get_attribute("lucky"))[0] index2 = where(self.gridcells.get_attribute("filter"))[0] lucky = self.households.get_attribute("lucky") weight = repeat(self.gridcells.get_attribute("weight")[newaxis, :], n, axis=0) for i in range(n): weight[i, :] += lucky[i] for icc in [0, 1]: sampler_ret = weighted_sampler().run(dataset1=self.households, dataset2=self.gridcells, index1=index1, index2=index2, sample_size=sample_size, weight=weight, include_chosen_choice=icc) sampled_index = sampler_ret.get_2d_index() chosen_choices = UNPLACED_ID * ones(index1.size, dtype=DTYPE) where_chosen = where(sampler_ret.get_attribute("chosen_choice")) chosen_choices[where_chosen[0]] = where_chosen[1] self.assertEqual(sampled_index.shape, (index1.size, sample_size)) if icc: placed_agents_index = self.gridcells.try_get_id_index( self.households.get_attribute("grid_id")[index1], UNPLACED_ID) chosen_choice_index = resize(array([UNPLACED_ID], dtype=DTYPE), index1.shape) w = where(chosen_choices >= 0)[0] chosen_choice_index[w] = sampled_index[w, chosen_choices[w]] self.assert_( alltrue(equal(placed_agents_index, chosen_choice_index))) sampled_index = sampled_index[:, 1:] self.assert_( alltrue( lookup(sampled_index.ravel(), index2, index_if_not_found=UNPLACED_ID) != UNPLACED_ID)) for j in range(sample_size): self.assert_( all(not_equal(weight[j, sampled_index[j, :]], 0.0)))
class Test(opus_unittest.OpusTestCase): def setUp(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='households', table_data={ 'household_id': arange(10)+1, # 'household_id':array([1, 2, 3, 4, 5, 6, 7, 8]), # 'income' :array([1, 3, 2, 1, 3, 8, 5, 4]), # #'category_id' :array([1, 2, 2, 1, 2, 3, 3, 2]), # 'building_id' :array([1, 2, 4, 3, 3, 2, 4, 2]), ##'large_area_id':array([1, 1, 2, 3, 3, 1, 2, 1]), # 'grid_id': arange(-1, 9, 1)+1, 'lucky':array([1,0,1, 0,1,1, 1,1,0, 0]) } ) storage.write_table(table_name='gridcells', table_data={ #'building_id': array([1, 2, 3, 4]), #'large_area_id': array([1, 1, 3, 2]), 'grid_id': arange(15)+1, 'filter':array([0,1,1, 1,1,1, 1,1,1, 0,1,0, 1,1,1]), 'weight':array([0.1,9,15, 2,5,1, 6,2.1,.3, 4,3,1, 10,8,7]) } ) dataset_pool = SessionConfiguration(in_storage=storage).get_dataset_pool() #create households self.households = Dataset(in_storage=storage, in_table_name='households', id_name="household_id", dataset_name="household") # create gridcells self.gridcells = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name="gridcell") dataset_pool.replace_dataset('household', self.households) dataset_pool.replace_dataset('gridcell', self.gridcells) def test_1(self): """""" sample_size = 5 # check the individual gridcells # This is a stochastic model, so it may legitimately fail occassionally. index1 = where(self.households.get_attribute("lucky"))[0] #index2 = where(self.gridcells.get_attribute("filter"))[0] weight=self.gridcells.get_attribute("weight") estimation_config = {"agent_category_definition":["household.lucky"], "choice_category_definition":["gridcell.filter+1"] } for icc in [0,1]: #include_chosen_choice? #icc = sample([0,1],1) sampler_ret = weighted_sampler_by_category().run(dataset1=self.households, dataset2=self.gridcells, index1=index1, sample_size=sample_size, include_chosen_choice=icc, resources=estimation_config) # get results sampled_index = sampler_ret.get_2d_index() chosen_choices = UNPLACED_ID * ones(index1.size, dtype="int32") where_chosen = where(sampler_ret.get_attribute("chosen_choice")) chosen_choices[where_chosen[0]]=where_chosen[1] self.assertEqual(sampled_index.shape, (index1.size, sample_size)) if icc: placed_agents_index = self.gridcells.try_get_id_index( self.households.get_attribute("grid_id")[index1],UNPLACED_ID) chosen_choice_index = resize(array([UNPLACED_ID], dtype="int32"), index1.shape) w = where(chosen_choices>=0)[0] # for 64 bit machines, need to coerce the type to int32 -- on a # 32 bit machine the astype(int32) doesn't do anything chosen_choice_index[w] = sampled_index[w, chosen_choices[w]].astype(int32) self.assert_( alltrue(equal(placed_agents_index, chosen_choice_index)) ) sampled_index = sampled_index[:,1:] self.assert_( alltrue(lookup(sampled_index.ravel(), arange(self.gridcells.size()), index_if_not_found=UNPLACED_ID)!=UNPLACED_ID) ) self.assert_( all(not_equal(weight[sampled_index], 0.0)) )
class Test(opus_unittest.OpusTestCase): def setUp(self): storage = StorageFactory().get_storage('dict_storage') storage.write_table(table_name='households', table_data={ 'household_id': arange(10)+1, 'grid_id': arange(-1, 9, 1)+1, 'lucky':array([1,0,1, 0,1,1, 1,1,0, 0]) } ) storage.write_table(table_name='gridcells', table_data={ 'grid_id': arange(15)+1, 'filter':array([0,1,1, 1,1,1, 1,1,1, 0,1,0, 1,1,1]), 'weight':array([0.1,9,15, 2,5,1, 6,2.1,.3, 4,3,1, 10,8,7]) } ) #create households self.households = Dataset(in_storage=storage, in_table_name='households', id_name="household_id", dataset_name="household") # create gridcells self.gridcells = Dataset(in_storage=storage, in_table_name='gridcells', id_name="grid_id", dataset_name="gridcell") def test_1d_weight_array(self): """""" sample_size = 5 # check the individual gridcells # This is a stochastic model, so it may legitimately fail occassionally. index1 = where(self.households.get_attribute("lucky"))[0] index2 = where(self.gridcells.get_attribute("filter"))[0] weight=self.gridcells.get_attribute("weight") for icc in [0,1]: #include_chosen_choice? #icc = sample([0,1],1) sampler_ret = weighted_sampler().run(dataset1=self.households, dataset2=self.gridcells, index1=index1, index2=index2, sample_size=sample_size, weight="weight",include_chosen_choice=icc) # get results sampled_index = sampler_ret.get_2d_index() chosen_choices = UNPLACED_ID * ones(index1.size, dtype="int32") where_chosen = where(sampler_ret.get_attribute("chosen_choice")) chosen_choices[where_chosen[0]]=where_chosen[1] sample_results = sampled_index, chosen_choices sampled_index = sample_results[0] self.assertEqual(sampled_index.shape, (index1.size, sample_size)) if icc: placed_agents_index = self.gridcells.try_get_id_index( self.households.get_attribute("grid_id")[index1],UNPLACED_ID) chosen_choice_index = resize(array([UNPLACED_ID], dtype="int32"), index1.shape) w = where(chosen_choices>=0)[0] # for 64 bit machines, need to coerce the type to int32 -- on a # 32 bit machine the astype(int32) doesn't do anything chosen_choice_index[w] = sampled_index[w, chosen_choices[w]].astype(int32) self.assert_( alltrue(equal(placed_agents_index, chosen_choice_index)) ) sampled_index = sampled_index[:,1:] self.assert_( alltrue(lookup(sampled_index.ravel(), index2, index_if_not_found=UNPLACED_ID)!=UNPLACED_ID) ) self.assert_( all(not_equal(weight[sampled_index], 0.0)) ) def test_2d_weight_array(self): #2d weight sample_size = 5 n = self.households.size() index1 = where(self.households.get_attribute("lucky"))[0] index2 = where(self.gridcells.get_attribute("filter"))[0] lucky = self.households.get_attribute("lucky") weight = repeat(self.gridcells.get_attribute("weight")[newaxis, :], n, axis=0) for i in range(n): weight[i,:] += lucky[i] for icc in [0,1]: sampler_ret = weighted_sampler().run(dataset1=self.households, dataset2=self.gridcells, index1=index1, index2=index2, sample_size=sample_size, weight=weight,include_chosen_choice=icc) sampled_index = sampler_ret.get_2d_index() chosen_choices = UNPLACED_ID * ones(index1.size, dtype="int32") where_chosen = where(sampler_ret.get_attribute("chosen_choice")) chosen_choices[where_chosen[0]]=where_chosen[1] self.assertEqual(sampled_index.shape, (index1.size, sample_size)) if icc: placed_agents_index = self.gridcells.try_get_id_index( self.households.get_attribute("grid_id")[index1],UNPLACED_ID) chosen_choice_index = resize(array([UNPLACED_ID], dtype="int32"), index1.shape) w = where(chosen_choices>=0)[0] chosen_choice_index[w] = sampled_index[w, chosen_choices[w]].astype(int32) self.assert_( alltrue(equal(placed_agents_index, chosen_choice_index)) ) sampled_index = sampled_index[:,1:] self.assert_( alltrue(lookup(sampled_index.ravel(), index2, index_if_not_found=UNPLACED_ID)!=UNPLACED_ID) ) for j in range(sample_size): self.assert_( all(not_equal(weight[j, sampled_index[j,:]], 0.0)) )