def test_import_file(self): tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user) tasks.map_data(self.import_file.pk) ps = PropertyState.objects.filter(pm_property_id='2264').first() ps.promote(self.cycle) # should only be 11 unmatched_properties because one was promoted. ps = self.import_file.find_unmatched_property_states() self.assertEqual(len(ps), 13)
def test_save_raw_data(self): """Save information in extra_data, set other attrs.""" with patch.object(ImportFile, 'cache_first_rows', return_value=None): tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) raw_saved = PropertyState.objects.filter( import_file=self.import_file, ).latest('id') self.assertDictEqual(raw_saved.extra_data, self.fake_extra_data) self.assertEqual(raw_saved.organization, self.org)
def test_match_buildings(self): """ case A (one property <-> one tax lot) """ tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user) tasks.map_data(self.import_file.pk) # Check to make sure all the properties imported ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file, ) self.assertEqual(len(ps), 14) # Check to make sure the taxlots were imported ts = TaxLotState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file, ) # self.assertEqual(len(ts), 10) # 10 unique taxlots after duplicates and delimeters # Check a single case of the taxlotstate ts = TaxLotState.objects.filter(jurisdiction_tax_lot_id='1552813').first() self.assertEqual(ts.jurisdiction_tax_lot_id, '1552813') self.assertEqual(ts.address_line_1, None) self.assertEqual(ts.extra_data["data_008"], 1) # Check a single case of the propertystate ps = PropertyState.objects.filter(pm_property_id='2264') self.assertEqual(len(ps), 1) ps = ps.first() self.assertEqual(ps.pm_property_id, '2264') self.assertEqual(ps.address_line_1, '50 Willow Ave SE') self.assertEqual('data_007' in ps.extra_data.keys(), True) self.assertEqual('data_008' in ps.extra_data.keys(), False) self.assertEqual(ps.extra_data["data_007"], 'a') # verify that the lot_number has the tax_lot information. For this case it is one-to-one self.assertEqual(ps.lot_number, ts.jurisdiction_tax_lot_id) tasks.match_buildings(self.import_file.id, self.user.id) self.assertEqual(TaxLot.objects.count(), 10) qry = PropertyView.objects.filter(state__custom_id_1='7') self.assertEqual(qry.count(), 1) state = qry.first().state self.assertEqual(state.address_line_1, "12 Ninth Street") self.assertEqual(state.property_name, "Grange Hall")
def test_multiple_id_matches(self): tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user) tasks.map_data(self.import_file.pk) # verify that there are no properties listed as canonical property_states = tasks.list_canonical_property_states(self.org) self.assertEqual(len(property_states), 0) # promote two properties ps = PropertyState.objects.filter(custom_id_1='13') ps_test = ps.first() ps_test_2 = ps.last() for p in ps: p.promote(self.cycle) # from seed.utils.generic import pp # pp(p) property_states = tasks.list_canonical_property_states(self.org) self.assertEqual(len(property_states), 2) # no arguments passed should return no results matches = tasks.query_property_matches(property_states, None, None) self.assertEqual(len(matches), 0) # should return 2 properties matches = tasks.query_property_matches(property_states, None, '13') self.assertEqual(len(matches), 2) self.assertEqual(matches[0], ps_test) self.assertEqual(matches[1], ps_test_2) # should return only the second property matches = tasks.query_property_matches(property_states, '2342', None) self.assertEqual(len(matches), 1) self.assertEqual(matches[0], ps_test_2) # should return both properties, the first one should be the pm match, i.e. the first prop matches = tasks.query_property_matches(property_states, '481516', '13') self.assertEqual(len(matches), 2) self.assertEqual(matches[0], ps_test) self.assertEqual(matches[1], ps_test_2) # if passing in the second pm then it will not be the first matches = tasks.query_property_matches(property_states, '2342', '13') self.assertEqual(len(matches), 2) self.assertEqual(matches[1], ps_test_2) # pass the pm id into the custom id. it should still return the correct buildings. # not sure that this is the right behavior, but this is what it does, so just testing. matches = tasks.query_property_matches(property_states, None, '2342') self.assertEqual(len(matches), 1) self.assertEqual(matches[0], ps_test_2) matches = tasks.query_property_matches(property_states, '13', None) self.assertEqual(len(matches), 2) self.assertEqual(matches[0], ps_test) self.assertEqual(matches[1], ps_test_2)
def test_cached_first_row_order(self): """Tests to make sure the first row is saved in the correct order. It should be the order of the headers in the original file.""" with patch.object(ImportFile, 'cache_first_rows', return_value=None): tasks._save_raw_data( self.import_file.pk, 'fake_cache_key', 1 ) expected_first_row = u"Property Id|#*#|Property Name|#*#|Year Ending|#*#|Property Floor Area (Buildings and Parking) (ft2)|#*#|Address 1|#*#|Address 2|#*#|City|#*#|State/Province|#*#|Postal Code|#*#|Year Built|#*#|ENERGY STAR Score|#*#|Site EUI (kBtu/ft2)|#*#|Total GHG Emissions (MtCO2e)|#*#|Weather Normalized Site EUI (kBtu/ft2)|#*#|National Median Site EUI (kBtu/ft2)|#*#|Source EUI (kBtu/ft2)|#*#|Weather Normalized Source EUI (kBtu/ft2)|#*#|National Median Source EUI (kBtu/ft2)|#*#|Parking - Gross Floor Area (ft2)|#*#|Organization|#*#|Generation Date|#*#|Release Date" # NOQA import_file = ImportFile.objects.get(pk=self.import_file.pk) first_row = import_file.cached_first_row self.assertEqual(first_row, expected_first_row)
def test_mapping_no_properties(self): # update the mappings to not include any taxlot tables in the data for m in self.fake_mappings: if m["to_table_name"] == 'PropertyState': m["to_table_name"] = 'TaxLotState' tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user) tasks.map_data(self.import_file.pk) # make sure that no taxlot objects were created. the 12 here are the import extra_data. ps = PropertyState.objects.all() self.assertEqual(len(ps), 14) # make sure that the new data was loaded correctly ts = TaxLotState.objects.filter(address_line_1='50 Willow Ave SE').first() self.assertEqual(ts.extra_data['site_eui'], 125)
def test_mapping_tax_lots_only(self): # update the mappings to not include any taxlot tables in the data new_mappings = copy.deepcopy(self.fake_mappings) for m in new_mappings: if m["to_table_name"] == 'PropertyState': m["to_table_name"] = 'TaxLotState' tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(new_mappings, self.org, self.user, self.import_file.pk) tasks.map_data(self.import_file.pk) # make sure that no taxlot objects were created. the 12 here are the import extra_data. ps = PropertyState.objects.all() self.assertEqual(len(ps), 14) # make sure that the new data was loaded correctly ts = TaxLotState.objects.filter(address_line_1='50 Willow Ave SE').first() self.assertEqual(ts.extra_data['site_eui'], 125)
def setUp(self): filename = getattr(self, 'filename', 'example-data-properties-duplicates.xlsx') import_file_source_type = ASSESSED_RAW self.fake_mappings = FAKE_MAPPINGS['portfolio'] self.fake_extra_data = FAKE_EXTRA_DATA self.fake_row = FAKE_ROW selfvars = self.set_up(import_file_source_type) self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars filepath = osp.join(osp.dirname(__file__), 'data', filename) self.import_file.file = SimpleUploadedFile(name=filename, content=open( filepath, 'rb').read()) self.import_file.save() tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) tasks.map_data(self.import_file.pk)
def test_match_buildings(self): """ case B (many property <-> one tax lot) """ tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) # Set remap to True because for some reason this file id has been imported before. tasks.map_data(self.import_file.pk, True) # Check to make sure all the properties imported ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file, ) self.assertEqual(len(ps), 14) # Check to make sure the tax lots were imported ts = TaxLotState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file, ) self.assertEqual(len(ts), 18) # verify that the lot_number has the tax_lot information. For this case it is one-to-many p_test = PropertyState.objects.filter( pm_property_id='5233255', organization=self.org, data_state=DATA_STATE_MAPPING, import_file=self.import_file, ).first() self.assertEqual(p_test.lot_number, "333/66555;333/66125;333/66148") tasks.match_buildings(self.import_file.id) # make sure the the property only has one tax lot and vice versa tlv = TaxLotView.objects.filter( state__jurisdiction_tax_lot_id='11160509', cycle=self.cycle) self.assertEqual(len(tlv), 1) tlv = tlv[0] properties = tlv.property_states() self.assertEqual(len(properties), 3)
def test_mapping_no_taxlot(self): # update the mappings to not include any taxlot tables in the data # note that save_data reads in from the propertystate table, so that will always # have entries in the db (for now). for m in self.fake_mappings: if m["to_table_name"] == 'TaxLotState': m["to_table_name"] = 'PropertyState' tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user) tasks.map_data(self.import_file.pk) # make sure that no taxlot objects were created ts = TaxLotState.objects.all() self.assertEqual(len(ts), 0) # make sure that the new data was loaded correctly ps = PropertyState.objects.filter(address_line_1='2700 Welstone Ave NE')[0] self.assertEqual(ps.site_eui, 1202) self.assertEqual(ps.extra_data['jurisdiction_tax_lot_id'], '11160509')
def test_mapping(self): tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user) tasks.map_data(self.import_file.pk) # There are a total of 18 tax lot ids in the import file ts = TaxLotState.objects.all() self.assertEqual(len(ts), 18) # make sure that the new data was loaded correctly and that the lot_number was set # appropriately ps = PropertyState.objects.filter(address_line_1='2700 Welstone Ave NE')[0] self.assertEqual(ps.site_eui, 1202) self.assertEqual(ps.lot_number, '11160509') ps = PropertyState.objects.filter(address_line_1='521 Elm Street')[0] self.assertEqual(ps.site_eui, 1358) # The lot_number should also have the normalized code run, then re-delimited self.assertEqual(ps.lot_number, '33366555;33366125;33366148')
def test_mapping(self): tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) tasks.map_data(self.import_file.pk) # There are a total of 18 tax lot ids in the import file ts = TaxLotState.objects.all() self.assertEqual(len(ts), 18) # make sure that the new data was loaded correctly and that the lot_number was set # appropriately ps = PropertyState.objects.filter(address_line_1='2700 Welstone Ave NE')[0] self.assertEqual(ps.site_eui, 1202) self.assertEqual(ps.lot_number, '11160509') ps = PropertyState.objects.filter(address_line_1='521 Elm Street')[0] self.assertEqual(ps.site_eui, 1358) # The lot_number should also have the normalized code run, then re-delimited self.assertEqual(ps.lot_number, '333/66555;333/66125;333/66148')
def test_single_id_matches(self): tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user) tasks.map_data(self.import_file.pk) # verify that there are no properties listed as canonical property_states = tasks.list_canonical_property_states(self.org) self.assertEqual(len(property_states), 0) # promote a properties ps = PropertyState.objects.filter(pm_property_id='2264').first() ps.promote(self.cycle) property_states = tasks.list_canonical_property_states(self.org) self.assertEqual(len(property_states), 1) matches = tasks.query_property_matches(property_states, None, None) self.assertEqual(len(matches), 0) matches = tasks.query_property_matches(property_states, '2264', None) self.assertEqual(len(matches), 1) self.assertEqual(matches[0], ps)
def test_mapping_properties_only(self): # update the mappings to not include any taxlot tables in the data # note that save_data reads in from the propertystate table, so that will always # have entries in the db (for now). new_mappings = copy.deepcopy(self.fake_mappings) for m in new_mappings: if m["to_table_name"] == 'TaxLotState': m["to_table_name"] = 'PropertyState' tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(new_mappings, self.org, self.user, self.import_file.pk) tasks.map_data(self.import_file.pk) # make sure that no taxlot objects were created ts = TaxLotState.objects.all() self.assertEqual(len(ts), 0) # make sure that the new data was loaded correctly ps = PropertyState.objects.filter(address_line_1='2700 Welstone Ave NE')[0] self.assertEqual(ps.site_eui, 1202) self.assertEqual(ps.extra_data['jurisdiction_tax_lot_id'], '11160509')
def setUp(self): filename = getattr(self, 'filename', 'example-data-properties.xlsx') self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio']) selfvars = self.set_up(ASSESSED_RAW) self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars filepath = osp.join(osp.dirname(__file__), 'data', filename) self.import_file.file = SimpleUploadedFile(name=filename, content=open( filepath, 'rb').read()) self.import_file.save() tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.id) tasks.map_data(self.import_file.pk) tasks.match_buildings(self.import_file.id) # import second file that is currently the same, but should be slightly different filename_2 = getattr(self, 'filename', 'example-data-properties-small-changes.xlsx') _, self.import_file_2 = self.create_import_file( self.user, self.org, self.cycle) filepath = osp.join(osp.dirname(__file__), 'data', filename_2) self.import_file_2.file = SimpleUploadedFile(name=filename_2, content=open( filepath, 'rb').read()) self.import_file_2.save() tasks._save_raw_data(self.import_file_2.pk, 'fake_cache_key_2', 1) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file_2.id) tasks.map_data(self.import_file_2.pk) tasks.match_buildings(self.import_file_2.id) # for api tests user_details = { 'username': '******', 'password': '******', } self.client.login(**user_details)
def test_demo_v2(self): tasks._save_raw_data(self.import_file_tax_lot.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_taxlot_mappings, self.org, self.user) Column.create_mappings(self.fake_portfolio_mappings, self.org, self.user) tasks.map_data(self.import_file_tax_lot.pk) # Check to make sure the taxlots were imported ts = TaxLotState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_tax_lot, ) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) self.assertEqual(len(ps), 0) self.assertEqual(len(ts), 9) tasks.match_buildings(self.import_file_tax_lot.id) # Check a single case of the taxlotstate self.assertEqual(TaxLotState.objects.filter(address_line_1='050 Willow Ave SE').count(), 1) self.assertEqual( TaxLotView.objects.filter(state__address_line_1='050 Willow Ave SE').count(), 1 ) self.assertEqual(TaxLotView.objects.count(), 9) # Import the property data tasks._save_raw_data(self.import_file_property.pk, 'fake_cache_key', 1) tasks.map_data(self.import_file_property.pk) ts = TaxLotState.objects.filter( # data_state=DATA_STATE_MAPPING, # Look at all taxlotstates organization=self.org, import_file=self.import_file_tax_lot, ) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) self.assertEqual(len(ts), 9) self.assertEqual(len(ps), 14) tasks.match_buildings(self.import_file_property.id) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) # there should not be any properties left in the mapping state self.assertEqual(len(ps), 0) # psv = PropertyView.objects.filter(state__organization=self.org) # self.assertEqual(len(psv), 12) # tlv = TaxLotView.objects.filter(state__organization=self.org) # self.assertEqual(len(tlv), 9) self.assertEqual(PropertyView.objects.filter(state__organization=self.org, state__pm_property_id='2264').count(), 1) pv = PropertyView.objects.filter(state__organization=self.org, state__pm_property_id='2264').first() self.assertEqual(pv.state.property_name, 'University Inn') self.assertEqual(pv.state.address_line_1, '50 Willow Ave SE')
def test_demo_v2(self): tasks._save_raw_data(self.import_file_tax_lot.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_taxlot_mappings, self.org, self.user) Column.create_mappings(self.fake_portfolio_mappings, self.org, self.user) tasks.map_data(self.import_file_tax_lot.pk) # Check to make sure the taxlots were imported ts = TaxLotState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_tax_lot, ) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) self.assertEqual(len(ps), 0) self.assertEqual(len(ts), 9) tasks.match_buildings(self.import_file_tax_lot.id, self.user.id) # Check a single case of the taxlotstate self.assertEqual(TaxLotState.objects.filter(address_line_1='050 Willow Ave SE').count(), 1) self.assertEqual( TaxLotView.objects.filter(state__address_line_1='050 Willow Ave SE').count(), 1 ) self.assertEqual(TaxLotView.objects.count(), 9) # Import the property data tasks._save_raw_data(self.import_file_property.pk, 'fake_cache_key', 1) tasks.map_data(self.import_file_property.pk) ts = TaxLotState.objects.filter( # data_state=DATA_STATE_MAPPING, # Look at all taxlotstates organization=self.org, import_file=self.import_file_tax_lot, ) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) self.assertEqual(len(ts), 9) self.assertEqual(len(ps), 14) tasks.match_buildings(self.import_file_property.id, self.user.id) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) # there shouldn't be any properties left in the mapping state self.assertEqual(len(ps), 0) # psv = PropertyView.objects.filter(state__organization=self.org) # self.assertEqual(len(psv), 12) # tlv = TaxLotView.objects.filter(state__organization=self.org) # self.assertEqual(len(tlv), 9) self.assertEqual(PropertyView.objects.filter(state__organization=self.org, state__pm_property_id='2264').count(), 1) pv = PropertyView.objects.filter(state__organization=self.org, state__pm_property_id='2264').first() self.assertEqual(pv.state.property_name, 'University Inn') self.assertEqual(pv.state.address_line_1, '50 Willow Ave SE')