def setUp(self): # for now just import some test data. I'd rather create fake data... next time. filename = getattr(self, 'filename', 'example-data-properties.xlsx') self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio']) selfvars = self.set_up(ASSESSED_RAW) self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars filepath = osp.join(osp.dirname(__file__), '..', 'data', filename) self.import_file.file = SimpleUploadedFile(name=filename, content=open( filepath, 'rb').read()) self.import_file.save() tasks.save_raw_data(self.import_file.pk) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.id) tasks.map_data(self.import_file.pk) tasks.match_buildings(self.import_file.id) # import second file with tax lot information filename_2 = getattr(self, 'filename', 'example-data-taxlots.xlsx') self.fake_mappings = copy.copy(FAKE_MAPPINGS['taxlot']) _, self.import_file_2 = self.create_import_file( self.user, self.org, self.cycle) filepath = osp.join(osp.dirname(__file__), '..', 'data', filename_2) self.import_file_2.file = SimpleUploadedFile(name=filename_2, content=open( filepath, 'rb').read()) self.import_file_2.save() tasks.save_raw_data(self.import_file_2.pk) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.id) tasks.map_data(self.import_file_2.pk) tasks.match_buildings(self.import_file_2.id)
def test_no_matches_if_all_matching_criteria_is_None(self): """ Default matching criteria for PropertyStates are: - address_line_1 (substituted by normalized_address) - ubid - pm_property_id - custom_id_1 and all are set to None. """ base_details = { 'import_file_id': self.import_file.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create set of properties that won't match self.property_state_factory.get_property_state(**base_details) base_details['city'] = 'Denver' self.property_state_factory.get_property_state(**base_details) # set import_file mapping done so that matching can occur. self.import_file.mapping_done = True self.import_file.save() match_buildings(self.import_file.id) # 2 Property, 2 PropertyView, 2 PropertyStates - No merges self.assertEqual(Property.objects.count(), 2) self.assertEqual(PropertyView.objects.count(), 2) self.assertEqual(PropertyState.objects.count(), 2)
def test_duplicate_properties_identified(self): base_details = { 'address_line_1': '123 Match Street', 'import_file_id': self.import_file_1.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create property in first ImportFile ps_1 = self.property_state_factory.get_property_state(**base_details) self.import_file_1.mapping_done = True self.import_file_1.save() match_buildings(self.import_file_1.id) # Create duplicate property coming from second ImportFile base_details['import_file_id'] = self.import_file_2.id ps_2 = self.property_state_factory.get_property_state(**base_details) self.import_file_2.mapping_done = True self.import_file_2.save() match_buildings(self.import_file_2.id) # 1 Property, 1 PropertyViews, 2 PropertyStates self.assertEqual(Property.objects.count(), 1) self.assertEqual(PropertyView.objects.count(), 1) self.assertEqual(PropertyState.objects.count(), 2) # Be sure the first property is used in the -View and the second is marked for "deletion" self.assertEqual(PropertyView.objects.first().state_id, ps_1.id) self.assertEqual( PropertyState.objects.get(data_state=DATA_STATE_DELETE).id, ps_2.id)
def setUp(self): super(TestProperties, self).setUp() # for now just import some test data. I'd rather create fake data... next time. filename = getattr(self, 'filename', 'example-data-properties.xlsx') self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio']) selfvars = self.set_up(ASSESSED_RAW) self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars filepath = osp.join(osp.dirname(__file__), '..', 'data', filename) self.import_file.file = SimpleUploadedFile(name=filename, content=open( filepath, 'rb').read()) self.import_file.save() tasks.save_raw_data(self.import_file.pk) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.id) tasks.map_data(self.import_file.pk) tasks.match_buildings(self.import_file.id) # import second file that is currently the same, but should be slightly different filename_2 = getattr(self, 'filename', 'example-data-properties-small-changes.xlsx') _, self.import_file_2 = self.create_import_file( self.user, self.org, self.cycle) filepath = osp.join(osp.dirname(__file__), '..', 'data', filename_2) self.import_file_2.file = SimpleUploadedFile(name=filename_2, content=open( filepath, 'rb').read()) self.import_file_2.save() tasks.save_raw_data(self.import_file_2.pk) tasks.map_data(self.import_file_2.pk) tasks.match_buildings(self.import_file_2.id)
def test_match_properties_get_rolled_up_into_one_in_the_order_their_uploaded(self): """ The most recently uploaded should take precedence when merging states. If more than 2 states match each other, they are merged two at a time until one is remaining. Reminder, this is only for -States within an ImportFile. """ base_details = { 'address_line_1': '123 Match Street', 'import_file_id': self.import_file.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create first set of properties that match each other base_details['city'] = 'Philadelphia' self.property_state_factory.get_property_state(**base_details) base_details['city'] = 'Arvada' self.property_state_factory.get_property_state(**base_details) base_details['city'] = 'Golden' self.property_state_factory.get_property_state(**base_details) base_details['city'] = 'Denver' self.property_state_factory.get_property_state(**base_details) # set import_file mapping done so that matching can occur. self.import_file.mapping_done = True self.import_file.save() match_buildings(self.import_file.id) # 1 Property, 1 PropertyViews, 7 PropertyStates (4 imported, 3 merge results) self.assertEqual(Property.objects.count(), 1) self.assertEqual(PropertyView.objects.count(), 1) self.assertEqual(PropertyState.objects.count(), 7) self.assertEqual(PropertyView.objects.first().state.city, 'Denver')
def test_duplicate_taxlots_identified(self): base_details = { 'address_line_1': '123 Match Street', 'import_file_id': self.import_file.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create pair of properties that are exact duplicates self.taxlot_state_factory.get_taxlot_state(**base_details) self.taxlot_state_factory.get_taxlot_state(**base_details) # Create a non-matching, non-duplicate property base_details['address_line_1'] = '123 Different Ave' base_details['city'] = 'Denver' self.taxlot_state_factory.get_taxlot_state(**base_details) # set import_file mapping done so that matching can occur. self.import_file.mapping_done = True self.import_file.save() match_buildings(self.import_file.id) # 2 TaxLot, 2 TaxLotViews, 3 TaxLotState (1 flagged to be ignored) self.assertEqual(TaxLot.objects.count(), 2) self.assertEqual(TaxLotView.objects.count(), 2) self.assertEqual(TaxLotState.objects.count(), 3) self.assertEqual( TaxLotState.objects.filter(data_state=DATA_STATE_DELETE).count(), 1) # Make sure "deleted" -States are not found in the -Views deleted = TaxLotState.objects.get(data_state=DATA_STATE_DELETE) self.assertNotIn(deleted.id, TaxLotView.objects.values_list('state_id', flat=True))
def test_import_duplicates(self): # Check to make sure all the properties imported ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file, ) self.assertEqual(len(ps), 9) self.assertEqual(PropertyState.objects.filter(pm_property_id='2264').count(), 7) hashes = list(map(tasks.hash_state_object, ps)) self.assertEqual(len(hashes), 9) self.assertEqual(len(set(hashes)), 4) unique_property_states, _ = match.filter_duplicate_states(ps) self.assertEqual(len(unique_property_states), 4) tasks.match_buildings(self.import_file.id) self.assertEqual(Property.objects.count(), 3) self.assertEqual(PropertyView.objects.count(), 3) self.assertEqual(PropertyView.objects.filter(state__pm_property_id='2264').count(), 1) pv = PropertyView.objects.filter(state__pm_property_id='2264').first() self.assertEqual(pv.state.pm_property_id, '2264') self.assertEqual(pv.state.gross_floor_area, 12555 * ureg.feet ** 2) self.assertEqual(pv.state.energy_score, 75) self.assertEqual(TaxLot.objects.count(), 0) self.assertEqual(self.import_file.find_unmatched_property_states().count(), 2) self.assertEqual(self.import_file.find_unmatched_tax_lot_states().count(), 0)
def test_match_merge_in_cycle_ignores_taxlots_with_unpopulated_matching_criteria( self): base_details = { 'city': 'Golden', 'import_file_id': self.import_file_1.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create 3 non-duplicate taxlots with unpopulated matching criteria tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_details) base_details['city'] = 'Denver' tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_details) base_details['city'] = 'Philadelphia' tls_3 = self.taxlot_state_factory.get_taxlot_state(**base_details) self.import_file_1.mapping_done = True self.import_file_1.save() match_buildings(self.import_file_1.id) # Verify no match merges happen tls_1_view = TaxLotView.objects.get(state_id=tls_1.id) count_result, no_match_indicator = match_merge_in_cycle( tls_1_view.id, 'TaxLotState') self.assertEqual(count_result, 0) self.assertIsNone(no_match_indicator) self.assertEqual(TaxLot.objects.count(), 3) self.assertEqual(TaxLotState.objects.count(), 3) self.assertEqual(TaxLotView.objects.count(), 3) state_ids = list(TaxLotView.objects.all().values_list('state_id', flat=True)) self.assertCountEqual([tls_1.id, tls_2.id, tls_3.id], state_ids)
def setUp(self): filename = getattr(self, 'filename', 'example-data-properties.xlsx') self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio']) selfvars = self.set_up(ASSESSED_RAW) self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars self.import_file.load_import_file( osp.join(osp.dirname(__file__), 'data', filename)) tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.id) tasks.map_data(self.import_file.pk) tasks.match_buildings(self.import_file.id) # import second file that is currently the same, but should be slightly different filename_2 = getattr(self, 'filename', 'example-data-properties-small-changes.xlsx') _, self.import_file_2 = self.create_import_file( self.user, self.org, self.cycle) self.import_file_2.load_import_file( osp.join(osp.dirname(__file__), 'data', filename_2)) tasks._save_raw_data(self.import_file_2.pk, 'fake_cache_key_2', 1) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file_2.id) tasks.map_data(self.import_file_2.pk) tasks.match_buildings(self.import_file_2.id) # for api tests user_details = { 'username': '******', 'password': '******', } self.client.login(**user_details)
def setUp(self): # for now just import some test data. I'd rather create fake data... next time. filename = getattr(self, 'filename', 'example-data-properties.xlsx') self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio']) selfvars = self.set_up(ASSESSED_RAW) self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars self.import_file.load_import_file( osp.join(osp.dirname(__file__), '../data_importer/tests/data', filename)) tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.id) tasks.map_data(self.import_file.pk) tasks.match_buildings(self.import_file.id) # import second file that is currently the same, but should be slightly different filename_2 = getattr(self, 'filename', 'example-data-properties-small-changes.xlsx') _, self.import_file_2 = self.create_import_file( self.user, self.org, self.cycle) self.import_file_2.load_import_file( osp.join(osp.dirname(__file__), '../data_importer/tests/data', filename_2)) tasks._save_raw_data(self.import_file_2.pk, 'fake_cache_key_2', 1) tasks.map_data(self.import_file_2.pk) tasks.match_buildings(self.import_file_2.id)
def test_match_properties_rolls_up_multiple_existing_matches_in_id_order_if_they_exist(self): base_details = { 'pm_property_id': '123MatchID', 'city': 'Golden', 'import_file_id': self.import_file_1.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create 3 non-matching properties in first ImportFile ps_1 = self.property_state_factory.get_property_state(**base_details) base_details['pm_property_id'] = '789DifferentID' base_details['city'] = 'Denver' ps_2 = self.property_state_factory.get_property_state(**base_details) base_details['pm_property_id'] = '1337AnotherDifferentID' base_details['city'] = 'Philadelphia' ps_3 = self.property_state_factory.get_property_state(**base_details) self.import_file_1.mapping_done = True self.import_file_1.save() match_buildings(self.import_file_1.id) # Update -States to make the roll up order be 1, 3, 2 refreshed_ps_3 = PropertyState.objects.get(id=ps_3.id) refreshed_ps_3.pm_property_id = '123MatchID' refreshed_ps_3.save() refreshed_ps_2 = PropertyState.objects.get(id=ps_2.id) refreshed_ps_2.pm_property_id = '123MatchID' refreshed_ps_2.save() # Verify that none of the 3 have been merged self.assertEqual(Property.objects.count(), 3) self.assertEqual(PropertyState.objects.count(), 3) self.assertEqual(PropertyView.objects.count(), 3) # Import a property that will identify the first 3 as matches. base_details['import_file_id'] = self.import_file_2.id base_details['pm_property_id'] = '123MatchID' del base_details['city'] ps_4 = self.property_state_factory.get_property_state(**base_details) self.import_file_2.mapping_done = True self.import_file_2.save() match_buildings(self.import_file_2.id) # There should only be one PropertyView which is associated to new, merged -State self.assertEqual(PropertyView.objects.count(), 1) view = PropertyView.objects.first() self.assertNotIn(view.state_id, [ps_1.id, ps_2.id, ps_3.id, ps_4.id]) # It will have a -State having city as Denver self.assertEqual(view.state.city, 'Denver') # The corresponding log should be a System Match audit_log = PropertyAuditLog.objects.get(state_id=view.state_id) self.assertEqual(audit_log.name, 'System Match')
def test_match_properties_and_taxlots_with_ubid(self): # create an ImportFile for testing purposes. Seems like we would want to run this matching just on a # list of properties and taxlots. # # This emulates importing the following # UBID, Jurisdiction Tax Lot # ubid_100, lot_1 # ubid_101, lot_1 # ubid_110, lot_1 # ubid_111, lot_1 ids = [('ubid_100', 'lot_1'), ('ubid_101', 'lot_1'), ('ubid_110', 'lot_1'), ('ubid_111', 'lot_1')] for id in ids: self.property_state_factory.get_property_state( no_default_data=True, ubid=id[0], lot_number=id[1], import_file_id=self.import_file.id, data_state=DATA_STATE_MAPPING, ) self.taxlot_state_factory.get_taxlot_state( no_default_data=True, jurisdiction_tax_lot_id=ids[0][1], import_file_id=self.import_file.id, data_state=DATA_STATE_MAPPING, ) # for ps in PropertyState.objects.filter(organization=self.org): # print("%s -- %s -- %s" % (ps.lot_number, ps.import_file_id, ps.ubid)) # pv = PropertyView.objects.get(state=ps, cycle=self.cycle) # TaxLotProperty.objects.filter() # for tl in TaxLotState.objects.filter(organization=self.org): # print("%s -- %s" % (tl.import_file_id, tl.jurisdiction_tax_lot_id)) # set import_file mapping done so that matching can occur. self.import_file.mapping_done = True self.import_file.save() match_buildings(self.import_file.id) # for pv in PropertyView.objects.filter(state__organization=self.org): # print("%s -- %s" % (pv.state.ubid, pv.cycle)) # should only have 1 PropertyView and 4 taxlot views self.assertEqual( PropertyView.objects.filter(state__organization=self.org).count(), 4) self.assertEqual( TaxLotView.objects.filter(state__organization=self.org).count(), 1) tlv = TaxLotView.objects.filter(state__organization=self.org).first() # there should be 4 relationships in the TaxLotProperty associated with view, one each for the taxlots defined self.assertEqual( TaxLotProperty.objects.filter(taxlot_view_id=tlv).count(), 4)
def test_match_taxlots_rolls_up_multiple_existing_matches_in_id_order_if_they_exist( self): base_details = { 'jurisdiction_tax_lot_id': '123MatchID', 'city': 'Golden', 'import_file_id': self.import_file_1.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create 3 non-matching taxlots in first ImportFile tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_details) base_details['jurisdiction_tax_lot_id'] = '789DifferentID' base_details['city'] = 'Denver' tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_details) base_details['jurisdiction_tax_lot_id'] = '1337AnotherDifferentID' base_details['city'] = 'Philadelphia' tls_3 = self.taxlot_state_factory.get_taxlot_state(**base_details) self.import_file_1.mapping_done = True self.import_file_1.save() match_buildings(self.import_file_1.id) # Make all those states match TaxLotState.objects.filter(pk__in=[tls_2.id, tls_3.id]).update( jurisdiction_tax_lot_id='123MatchID') # Verify that none of the 3 have been merged self.assertEqual(TaxLot.objects.count(), 3) self.assertEqual(TaxLotState.objects.count(), 3) self.assertEqual(TaxLotView.objects.count(), 3) # Import a property that will identify the first 3 as matches. base_details['import_file_id'] = self.import_file_2.id base_details['jurisdiction_tax_lot_id'] = '123MatchID' del base_details['city'] tls_4 = self.taxlot_state_factory.get_taxlot_state(**base_details) self.import_file_2.mapping_done = True self.import_file_2.save() match_buildings(self.import_file_2.id) # There should only be one TaxLotView which is associated to new, merged -State self.assertEqual(TaxLotView.objects.count(), 1) view = TaxLotView.objects.first() self.assertNotIn(view.state_id, [tls_1.id, tls_2.id, tls_3.id, tls_4.id]) # It will have a -State having city as Philadelphia self.assertEqual(view.state.city, 'Philadelphia') # The corresponding log should be a System Match audit_log = TaxLotAuditLog.objects.get(state_id=view.state_id) self.assertEqual(audit_log.name, 'System Match')
def test_match_buildings(self): """ case A (one property <-> one tax lot) """ tasks.save_raw_data(self.import_file.pk) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) tasks.map_data(self.import_file.pk) # Check to make sure all the properties imported ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file, ) self.assertEqual(len(ps), 14) # Check to make sure the taxlots were imported ts = TaxLotState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file, ) self.assertEqual(len(ts), 18) # Check a single case of the taxlotstate ts = TaxLotState.objects.filter( jurisdiction_tax_lot_id='1552813').first() self.assertEqual(ts.jurisdiction_tax_lot_id, '1552813') self.assertEqual(ts.address_line_1, None) self.assertEqual(ts.extra_data["data_008"], 1) # Check a single case of the propertystate ps = PropertyState.objects.filter(pm_property_id='2264') self.assertEqual(len(ps), 1) ps = ps.first() self.assertEqual(ps.pm_property_id, '2264') self.assertEqual(ps.address_line_1, '50 Willow Ave SE') self.assertEqual('data_007' in ps.extra_data, True) self.assertEqual('data_008' in ps.extra_data, False) self.assertEqual(ps.extra_data["data_007"], 'a') # verify that the lot_number has the tax_lot information. For this case it is one-to-one self.assertEqual(ps.lot_number, ts.jurisdiction_tax_lot_id) tasks.match_buildings(self.import_file.id) self.assertEqual(TaxLot.objects.count(), 10) qry = PropertyView.objects.filter(state__custom_id_1='7') self.assertEqual(qry.count(), 1) state = qry.first().state self.assertEqual(state.address_line_1, "12 Ninth Street") self.assertEqual(state.property_name, "Grange Hall")
def test_match_properties_and_taxlots_with_address_no_lot_number(self): # create an ImportFile for testing purposes. Seems like we would want to run this matching just on a # list of properties and taxlots. # # This emulates importing the following # Address, Jurisdiction Tax Lot # 742 Evergreen Terrace, 100 # 742 Evergreen Terrace, 101 # 742 Evergreen Terrace, 110 # 742 Evergreen Terrace, 111 lot_numbers = '100;101;110;111' for lot_number in lot_numbers.split(';'): self.property_state_factory.get_property_state( address_line_1='742 Evergreen Terrace', lot_number=lot_number, import_file_id=self.import_file.id, data_state=DATA_STATE_MAPPING, ) self.taxlot_state_factory.get_taxlot_state( address_line_1=None, jurisdiction_tax_lot_id=lot_number, import_file_id=self.import_file.id, data_state=DATA_STATE_MAPPING, ) # for ps in PropertyState.objects.filter(organization=self.org): # print("%s -- %s -- %s" % (ps.lot_number, ps.import_file_id, ps.address_line_1)) # for tl in TaxLotState.objects.filter(organization=self.org): # print("%s -- %s" % (tl.import_file_id, tl.jurisdiction_tax_lot_id)) # set import_file mapping done so that matching can occur. self.import_file.mapping_done = True self.import_file.save() match_buildings(self.import_file.id) # for pv in PropertyView.objects.filter(state__organization=self.org): # print("%s -- %s" % (pv.state, pv.cycle)) # should only have 1 PropertyView and 4 taxlot views self.assertEqual( PropertyView.objects.filter(state__organization=self.org).count(), 1) self.assertEqual( TaxLotView.objects.filter(state__organization=self.org).count(), 4) pv = PropertyView.objects.filter(state__organization=self.org).first() # there should be 4 relationships in the TaxLotProperty associated with view, one each for the taxlots defined self.assertEqual( TaxLotProperty.objects.filter(property_view_id=pv).count(), 4)
def test_match_buildings(self): """ case A (one property <-> one tax lot) """ tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user) tasks.map_data(self.import_file.pk) # Check to make sure all the properties imported ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file, ) self.assertEqual(len(ps), 14) # Check to make sure the taxlots were imported ts = TaxLotState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file, ) # self.assertEqual(len(ts), 10) # 10 unique taxlots after duplicates and delimeters # Check a single case of the taxlotstate ts = TaxLotState.objects.filter(jurisdiction_tax_lot_id='1552813').first() self.assertEqual(ts.jurisdiction_tax_lot_id, '1552813') self.assertEqual(ts.address_line_1, None) self.assertEqual(ts.extra_data["data_008"], 1) # Check a single case of the propertystate ps = PropertyState.objects.filter(pm_property_id='2264') self.assertEqual(len(ps), 1) ps = ps.first() self.assertEqual(ps.pm_property_id, '2264') self.assertEqual(ps.address_line_1, '50 Willow Ave SE') self.assertEqual('data_007' in ps.extra_data.keys(), True) self.assertEqual('data_008' in ps.extra_data.keys(), False) self.assertEqual(ps.extra_data["data_007"], 'a') # verify that the lot_number has the tax_lot information. For this case it is one-to-one self.assertEqual(ps.lot_number, ts.jurisdiction_tax_lot_id) tasks.match_buildings(self.import_file.id, self.user.id) self.assertEqual(TaxLot.objects.count(), 10) qry = PropertyView.objects.filter(state__custom_id_1='7') self.assertEqual(qry.count(), 1) state = qry.first().state self.assertEqual(state.address_line_1, "12 Ninth Street") self.assertEqual(state.property_name, "Grange Hall")
def _create_propertyview(request, org, user, dataset_name): cycle = Cycle.objects.filter( organization=org).last() # might need to hardcode this dataset = ImportRecord.objects.get(name=dataset_name, super_organization=org) result = [{'City': request.GET['city'], 'State': request.GET['state']}] if 'street' in request.GET: result[0]['Address Line 1'] = request.GET['street'] else: result[0]['Address Line 1'] = request.GET['address_line_1'] if 'zipcode' in request.GET: result[0]['Postal Code'] = request.GET['zipcode'] else: result[0]['Postal Code'] = request.GET['postal_code'] if 'property_uid' in request.GET: result[0]['Custom ID 1'] = request.GET['property_uid'] file_pk = utils.save_and_load(user, dataset, cycle, result, "profile_data.csv") # save data resp = save_raw_data(file_pk) save_prog_key = resp['progress_key'] utils.wait_for_task(save_prog_key) # map data # save_column_mappings(file_id, col_mappings) #perform column mapping resp = map_data(file_pk) map_prog_key = resp['progress_key'] utils.wait_for_task(map_prog_key) resp = match_buildings(file_pk) # resp = geocode_buildings_task(file_pk) if (resp['status'] == 'error'): return resp match_prog_key = resp['progress_key'] utils.wait_for_task(match_prog_key) propertyview = utils.propertyview_find(request) return propertyview
def test_handle_id_matches_duplicate_data(self): """ Test for handle_id_matches behavior when matching duplicate data """ # TODO: Fix the PM, tax lot id, and custom ID fields in PropertyState bs_data = { 'pm_property_id': "2360", # 'tax_lot_id': '476/460', 'property_name': 'Garfield Complex', 'custom_id_1': "89", 'address_line_1': '12975 Database LN.', 'address_line_2': '', 'city': 'Cartoon City', 'postal_code': "54321", 'data_state': DATA_STATE_MAPPING, 'source_type': ASSESSED_BS, } # Setup mapped AS snapshot. PropertyState.objects.create( organization=self.org, import_file=self.import_file, **bs_data ) # Different file, but same ImportRecord. # Setup mapped PM snapshot. # Should be an identical match. new_import_file = ImportFile.objects.create(import_record=self.import_record, mapping_done=True) tasks.match_buildings(new_import_file.pk, self.user.pk) duplicate_import_file = ImportFile.objects.create( import_record=self.import_record, mapping_done=True ) PropertyState.objects.create( organization=self.org, import_file=duplicate_import_file, **bs_data )
def test_handle_id_matches_duplicate_data(self): """ Test for handle_id_matches behavior when matching duplicate data """ # TODO: Fix the PM, tax lot id, and custom ID fields in PropertyState bs_data = { 'pm_property_id': "2360", # 'tax_lot_id': '476/460', 'property_name': 'Garfield Complex', 'custom_id_1': "89", 'address_line_1': '12975 Database LN.', 'address_line_2': '', 'city': 'Cartoon City', 'postal_code': "54321", 'data_state': DATA_STATE_MAPPING, 'source_type': ASSESSED_BS, } # Setup mapped AS snapshot. PropertyState.objects.create( organization=self.org, import_file=self.import_file, **bs_data ) # Different file, but same ImportRecord. # Setup mapped PM snapshot. # Should be an identical match. new_import_file = ImportFile.objects.create(import_record=self.import_record, mapping_done=True) tasks.match_buildings(new_import_file.pk) duplicate_import_file = ImportFile.objects.create( import_record=self.import_record, mapping_done=True ) PropertyState.objects.create( organization=self.org, import_file=duplicate_import_file, **bs_data )
def test_match_buildings(self): """ case B (many property <-> one tax lot) """ tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) # Set remap to True because for some reason this file id has been imported before. tasks.map_data(self.import_file.pk, True) # Check to make sure all the properties imported ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file, ) self.assertEqual(len(ps), 14) # Check to make sure the tax lots were imported ts = TaxLotState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file, ) self.assertEqual(len(ts), 18) # verify that the lot_number has the tax_lot information. For this case it is one-to-many p_test = PropertyState.objects.filter( pm_property_id='5233255', organization=self.org, data_state=DATA_STATE_MAPPING, import_file=self.import_file, ).first() self.assertEqual(p_test.lot_number, "333/66555;333/66125;333/66148") tasks.match_buildings(self.import_file.id) # make sure the the property only has one tax lot and vice versa tlv = TaxLotView.objects.filter( state__jurisdiction_tax_lot_id='11160509', cycle=self.cycle) self.assertEqual(len(tlv), 1) tlv = tlv[0] properties = tlv.property_states() self.assertEqual(len(properties), 3)
def setUp(self): data_importer_data_dir = os.path.join(os.path.dirname(__file__), '..', 'data_importer', 'tests', 'data') filename = getattr(self, 'filename', 'example-data-properties.xlsx') self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio']) selfvars = self.set_up(ASSESSED_RAW) self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars filepath = os.path.join(data_importer_data_dir, filename) self.import_file.file = SimpleUploadedFile(name=filename, content=open( filepath, 'rb').read()) self.import_file.save() tasks.save_raw_data(self.import_file.pk) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.id) tasks.map_data(self.import_file.pk) tasks.match_buildings(self.import_file.id) # import second file that is currently the same, but should be slightly different filename_2 = getattr(self, 'filename', 'example-data-properties-small-changes.xlsx') _, self.import_file_2 = self.create_import_file( self.user, self.org, self.cycle) filepath = os.path.join(data_importer_data_dir, filename_2) self.import_file_2.file = SimpleUploadedFile(name=filename_2, content=open( filepath, 'rb').read()) self.import_file_2.save() tasks.save_raw_data(self.import_file_2.pk) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file_2.id) tasks.map_data(self.import_file_2.pk) tasks.match_buildings(self.import_file_2.id) # for api tests user_details = { 'username': '******', 'password': '******', } self.client.login(**user_details)
def test_match_properties_on_ubid(self): base_details = { 'ubid': '86HJPCWQ+2VV-1-3-2-3', 'import_file_id': self.import_file.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create set of properties that match each other self.property_state_factory.get_property_state(**base_details) base_details['city'] = 'Denver' self.property_state_factory.get_property_state(**base_details) # set import_file mapping done so that matching can occur. self.import_file.mapping_done = True self.import_file.save() match_buildings(self.import_file.id) # 1 Property, 1 PropertyView, 3 PropertyStates (2 imported, 1 merge result) self.assertEqual(Property.objects.count(), 1) self.assertEqual(PropertyView.objects.count(), 1) self.assertEqual(PropertyState.objects.count(), 3)
def test_match_taxlots_normalized_address_used_instead_of_address_line_1(self): base_details = { 'import_file_id': self.import_file.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create set of taxlots that have the same address_line_1 in slightly different format base_details['address_line_1'] = '123 Match Street' self.taxlot_state_factory.get_taxlot_state(**base_details) base_details['address_line_1'] = '123 match St.' base_details['city'] = 'Denver' self.taxlot_state_factory.get_taxlot_state(**base_details) # set import_file mapping done so that matching can occur. self.import_file.mapping_done = True self.import_file.save() match_buildings(self.import_file.id) # 1 TaxLot, 1 TaxLotView, 3 TaxLotStates (2 imported, 1 merge result) self.assertEqual(TaxLot.objects.count(), 1) self.assertEqual(TaxLotView.objects.count(), 1) self.assertEqual(TaxLotState.objects.count(), 3)
def start_system_matching(self, request, pk=None): """ Starts a background task to attempt automatic matching between buildings in an ImportFile with other existing buildings within the same org. --- type: status: required: true type: string description: either success or error progress_key: type: integer description: ID of background job, for retrieving job progress parameter_strategy: replace parameters: - name: pk description: Import file ID required: true paramType: path """ return match_buildings(pk, request.user.pk)
def test_check(self): # data quality check d = DataQualityCheck.retrieve(self.org) d.remove_all_rules() d.add_rule({ 'table_name': 'PropertyState', 'field': 'gross_floor_area', 'data_type': TYPE_NUMBER, 'rule_type': RULE_TYPE_DEFAULT, 'min': 100, 'max': 7000000, 'severity': SEVERITY_ERROR, 'units': 'square feet', }) d.add_rule({ 'table_name': 'PropertyState', 'field': 'recent_sale_date', 'data_type': TYPE_DATE, 'rule_type': RULE_TYPE_DEFAULT, 'min': 18890101, 'max': 20201231, 'severity': SEVERITY_ERROR, }) # create some status labels for testing sl_data = { 'name': 'year - old or future', 'super_organization': self.org } sl_year, _ = StatusLabel.objects.get_or_create(**sl_data) new_rule = { 'table_name': 'PropertyState', 'field': 'year_built', 'data_type': TYPE_YEAR, 'rule_type': RULE_TYPE_DEFAULT, 'min': 1700, 'max': 2019, 'severity': SEVERITY_ERROR, 'status_label': sl_year, } d.add_rule(new_rule) sl_data = { 'name': 'extra data ps float error', 'super_organization': self.org } sl_string, _ = StatusLabel.objects.get_or_create(**sl_data) new_rule = { 'table_name': 'PropertyState', 'field': 'extra_data_ps_alpha', 'data_type': TYPE_STRING, 'rule_type': RULE_TYPE_CUSTOM, 'text_match': 'alpha', 'severity': SEVERITY_ERROR, 'units': 'square feet', 'status_label': sl_string, } d.add_rule(new_rule) sl_data = { 'name': 'extra data ps string error', 'super_organization': self.org } sl_float, _ = StatusLabel.objects.get_or_create(**sl_data) new_rule = { 'table_name': 'PropertyState', 'field': 'extra_data_ps_float', 'data_type': TYPE_NUMBER, 'rule_type': RULE_TYPE_CUSTOM, 'min': 9999, 'max': 10001, 'severity': SEVERITY_ERROR, 'status_label': sl_float, } d.add_rule(new_rule) sl_data = { 'name': 'jurisdiction id does not match', 'super_organization': self.org } sl_jurid, _ = StatusLabel.objects.get_or_create(**sl_data) new_rule = { 'table_name': 'TaxLotState', 'field': 'jurisdiction_tax_lot_id', 'data_type': TYPE_STRING, 'rule_type': RULE_TYPE_CUSTOM, 'text_match': '1235', 'severity': SEVERITY_ERROR, 'status_label': sl_jurid, } d.add_rule(new_rule) sl_data = {'name': 'No meters present', 'super_organization': self.org} sl_ok_1, _ = StatusLabel.objects.get_or_create(**sl_data) new_rule = { 'table_name': 'PropertyState', 'field': 'Meters Present', 'data_type': TYPE_STRING, 'rule_type': RULE_TYPE_CUSTOM, 'text_match': 'OK', 'severity': SEVERITY_ERROR, 'status_label': sl_ok_1, } d.add_rule(new_rule) sl_data = { 'name': 'No 12 Consectutive Months', 'super_organization': self.org } sl_ok_2, _ = StatusLabel.objects.get_or_create(**sl_data) new_rule = { 'table_name': 'PropertyState', 'field': '12 Consectutive Months', 'data_type': TYPE_STRING, 'rule_type': RULE_TYPE_CUSTOM, 'text_match': 'OK', 'severity': SEVERITY_ERROR, 'status_label': sl_ok_2, } d.add_rule(new_rule) sl_data = {'name': 'No Monthly Data', 'super_organization': self.org} sl_ok_3, _ = StatusLabel.objects.get_or_create(**sl_data) new_rule = { 'table_name': 'PropertyState', 'field': 'Monthly Data', 'data_type': TYPE_STRING, 'rule_type': RULE_TYPE_CUSTOM, 'text_match': 'OK', 'severity': SEVERITY_ERROR, 'status_label': sl_ok_3, } d.add_rule(new_rule) # import data tasks.save_raw_data(self.import_file.id) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) tasks.map_data(self.import_file.id) tasks.match_buildings(self.import_file.id) qs = PropertyState.objects.filter( import_file=self.import_file, source_type=ASSESSED_BS, ).iterator() d.reset_results() d.check_data('PropertyState', qs) result = d.retrieve_result_by_address('4 Myrtle Parkway') res = [{ "severity": "error", "value": "27.0", "field": "extra_data_ps_float", "table_name": "PropertyState", "message": "Extra Data Ps Float out of range", "detailed_message": "Extra Data Ps Float [27.0] < 9999.0", "formatted_field": "Extra Data Ps Float" }, { "severity": "error", "value": "5.0", "field": "gross_floor_area", "table_name": "PropertyState", "message": "Gross Floor Area out of range", "detailed_message": "Gross Floor Area [5.0] < 100.0", "formatted_field": "Gross Floor Area" }] self.assertListEqual(result['data_quality_results'], res) result = d.retrieve_result_by_address('94 Oxford Hill') res = [{ "severity": "error", "value": "20000.0", "field": "extra_data_ps_float", "table_name": "PropertyState", "message": "Extra Data Ps Float out of range", "detailed_message": "Extra Data Ps Float [20000.0] > 10001.0", "formatted_field": "Extra Data Ps Float" }, { "severity": "error", "value": "1888-01-01 08:00:00", "field": "recent_sale_date", "table_name": "PropertyState", "message": "Recent Sale Date out of range", "detailed_message": "Recent Sale Date [1888-01-01 08:00:00] < 1889-01-01 00:00:00", "formatted_field": "Recent Sale Date" }] self.assertListEqual(result['data_quality_results'], res) result = d.retrieve_result_by_address("3 Portage Alley") res = [{ 'severity': u'error', 'value': 'beta', 'field': u'extra_data_ps_alpha', 'table_name': u'PropertyState', 'message': u'Extra Data Ps Alpha does not match expected value', 'detailed_message': u'Extra Data Ps Alpha [beta] does not contain "alpha"', 'formatted_field': u'Extra Data Ps Alpha' }] self.assertListEqual(result['data_quality_results'], res) # make sure that the label has been applied props = PropertyView.objects.filter( property__labels=sl_year).select_related('state') addresses = sorted([p.state.address_line_1 for p in props]) expected = sorted([u'84807 Buell Trail', u'1 International Road']) self.assertListEqual(expected, addresses) props = PropertyView.objects.filter( property__labels=sl_float).select_related('state') addresses = sorted([p.state.address_line_1 for p in props]) expected = sorted([u'4 Myrtle Parkway', u'94 Oxford Hill']) self.assertListEqual(expected, addresses) props = PropertyView.objects.filter( property__labels=sl_string).select_related('state') addresses = [p.state.address_line_1 for p in props] expected = [u'3 Portage Alley'] self.assertListEqual(expected, addresses) # Check tax lots qs = TaxLotState.objects.filter( import_file=self.import_file, ).iterator() d.reset_results() d.check_data('TaxLotState', qs) result = d.retrieve_result_by_tax_lot_id("1234") res = [{ "severity": "error", "value": "1234", "field": "jurisdiction_tax_lot_id", "table_name": "TaxLotState", "message": "Jurisdiction Tax Lot ID does not match expected value", "detailed_message": "Jurisdiction Tax Lot ID [1234] does not contain \"1235\"", "formatted_field": "Jurisdiction Tax Lot ID" }] self.assertListEqual(result['data_quality_results'], res) # verify labels taxlots = TaxLotView.objects.filter( taxlot__labels=sl_jurid).select_related('state') ids = [t.state.jurisdiction_tax_lot_id for t in taxlots] expected = '1234' self.assertEqual(expected, ids[0]) # Check multiple strings props = PropertyView.objects.filter( property__labels=sl_ok_1).select_related('state') addresses = [p.state.address_line_1 for p in props]
def test_taxlots(self): # Define matching values matching_jurisdiction_tax_lot_id = '11111' matching_address_line_1 = '123 Match Street' matching_ulid = '86HJPCWQ+2VV-1-3-2-3' matching_custom_id_1 = 'MatchingID12345' # For first file, create taxlots with no duplicates or matches base_details_file_1 = { 'import_file_id': self.import_file_1.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # No matching_criteria values self.taxlot_state_factory.get_taxlot_state(**base_details_file_1) # Build out taxlots with increasingly more matching_criteria values base_details_file_1[ 'jurisdiction_tax_lot_id'] = matching_jurisdiction_tax_lot_id self.taxlot_state_factory.get_taxlot_state(**base_details_file_1) base_details_file_1['address_line_1'] = matching_address_line_1 self.taxlot_state_factory.get_taxlot_state(**base_details_file_1) base_details_file_1['ulid'] = matching_ulid self.taxlot_state_factory.get_taxlot_state(**base_details_file_1) base_details_file_1['custom_id_1'] = matching_custom_id_1 self.taxlot_state_factory.get_taxlot_state(**base_details_file_1) self.import_file_1.mapping_done = True self.import_file_1.save() match_buildings(self.import_file_1.id) # Verify no duplicates/matched-merges yet counts = [ TaxLot.objects.count(), TaxLotState.objects.count(), TaxLotView.objects.count(), ] self.assertEqual([5, 5, 5], counts) """ For second file, create several taxlots that are one or many of the following: - duplicates amongst file_1 - duplicates amongst file_2 - matching amongst file_1 - matching amongst file_2 - completely new """ base_details_file_2 = { 'import_file_id': self.import_file_2.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create 2 duplicates of the 'No matching_criteria values' taxlots # (outcome: 2 additional -States, NO new TaxLot/-View) tls_1 = self.taxlot_state_factory.get_taxlot_state( **base_details_file_2) tls_2 = self.taxlot_state_factory.get_taxlot_state( **base_details_file_2) # Create 2 completely new taxlots with misaligned combinations of matching values # (outcome: 2 additional -States, 2 new TaxLot/-View) base_details_file_2['custom_id_1'] = matching_custom_id_1 tls_3 = self.taxlot_state_factory.get_taxlot_state( **base_details_file_2) base_details_file_2['ulid'] = matching_ulid tls_4 = self.taxlot_state_factory.get_taxlot_state( **base_details_file_2) # Create 3 taxlots - with 1 duplicate and 1 match within it's own file that will # eventually become 1 completely new property # (outcome: 4 additional -States, 1 new TaxLot/-View) base_details_file_2['address_line_1'] = matching_address_line_1 base_details_file_2['city'] = 'Denver' tls_5 = self.taxlot_state_factory.get_taxlot_state( **base_details_file_2) tls_6 = self.taxlot_state_factory.get_taxlot_state( **base_details_file_2) base_details_file_2['city'] = 'Golden' tls_7 = self.taxlot_state_factory.get_taxlot_state( **base_details_file_2) # Create 3 properties - with 1 duplicate and 1 match within it's own file that will # eventually match the last property in file_1 # (outcome: 5 additional -States, NO new TaxLot/-View) base_details_file_2[ 'jurisdiction_tax_lot_id'] = matching_jurisdiction_tax_lot_id base_details_file_2['state'] = 'Colorado' tls_8 = self.taxlot_state_factory.get_taxlot_state( **base_details_file_2) tls_9 = self.taxlot_state_factory.get_taxlot_state( **base_details_file_2) base_details_file_2['state'] = 'California' tls_10 = self.taxlot_state_factory.get_taxlot_state( **base_details_file_2) self.import_file_2.mapping_done = True self.import_file_2.save() match_buildings(self.import_file_2.id) self.assertEqual(8, TaxLot.objects.count()) self.assertEqual(8, TaxLotView.objects.count()) self.assertEqual(18, TaxLotState.objects.count()) tls_ids_of_deleted = TaxLotState.objects.filter( data_state=DATA_STATE_DELETE).values_list('id', flat=True).order_by('id') self.assertEqual([tls_1.id, tls_2.id, tls_6.id, tls_9.id], list(tls_ids_of_deleted)) tls_ids_of_merged_in_file = TaxLotState.objects.filter( data_state=DATA_STATE_MAPPING, merge_state=MERGE_STATE_UNKNOWN).values_list( 'id', flat=True).order_by('id') self.assertEqual([tls_5.id, tls_7.id, tls_8.id, tls_10.id], list(tls_ids_of_merged_in_file)) tls_ids_of_all_promoted = TaxLotView.objects.values_list('state_id', flat=True) self.assertIn(tls_3.id, tls_ids_of_all_promoted) self.assertIn(tls_4.id, tls_ids_of_all_promoted) rimport_file_2 = ImportFile.objects.get(pk=self.import_file_2.id) results = rimport_file_2.matching_results_data del results['progress_key'] expected = { 'import_file_records': None, # This is calculated in a separate process 'property_all_unmatched': 0, 'property_duplicates': 0, 'property_duplicates_of_existing': 0, 'property_unmatched': 0, 'tax_lot_all_unmatched': 10, 'tax_lot_duplicates': 3, 'tax_lot_duplicates_of_existing': 1, 'tax_lot_unmatched': 3, } self.assertEqual(results, expected)
def test_match_taxlots_if_all_default_fields_match(self): base_details = { 'address_line_1': '123 Match Street', 'import_file_id': self.import_file_1.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create property in first ImportFile tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_details) self.import_file_1.mapping_done = True self.import_file_1.save() match_buildings(self.import_file_1.id) # Create properties from second ImportFile, one matching existing PropertyState base_details['import_file_id'] = self.import_file_2.id base_details['city'] = 'Denver' tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_details) base_details['jurisdiction_tax_lot_id'] = '11111' base_details['city'] = 'Philadelphia' tls_3 = self.taxlot_state_factory.get_taxlot_state(**base_details) self.import_file_2.mapping_done = True self.import_file_2.save() match_buildings(self.import_file_2.id) # 2 TaxLot, 2 TaxLotViews, 4 TaxLotStates (3 imported, 1 merge result) self.assertEqual(TaxLot.objects.count(), 2) self.assertEqual(TaxLotView.objects.count(), 2) self.assertEqual(TaxLotState.objects.count(), 4) cities_from_views = [] tls_ids_from_views = [] for tlv in TaxLotView.objects.all(): cities_from_views.append(tlv.state.city) tls_ids_from_views.append(tlv.state_id) self.assertIn('Denver', cities_from_views) self.assertIn('Philadelphia', cities_from_views) self.assertIn(tls_3.id, tls_ids_from_views) self.assertNotIn(tls_1.id, tls_ids_from_views) self.assertNotIn(tls_2.id, tls_ids_from_views) # Refresh -States and check data_state and merge_state values rtls_1 = TaxLotState.objects.get(pk=tls_1.id) self.assertEqual(rtls_1.data_state, DATA_STATE_MATCHING) self.assertEqual(rtls_1.merge_state, MERGE_STATE_NEW) rtls_2 = TaxLotState.objects.get(pk=tls_2.id) self.assertEqual(rtls_2.data_state, DATA_STATE_MATCHING) self.assertEqual(rtls_2.merge_state, MERGE_STATE_UNKNOWN) tls_1_plus_2 = TaxLotState.objects.filter( jurisdiction_tax_lot_id__isnull=True, city='Denver', address_line_1='123 Match Street').exclude( data_state=DATA_STATE_MATCHING, merge_state=MERGE_STATE_UNKNOWN).get() self.assertEqual(tls_1_plus_2.data_state, DATA_STATE_MATCHING) self.assertEqual(tls_1_plus_2.merge_state, MERGE_STATE_MERGED) rtls_3 = TaxLotState.objects.get(pk=tls_3.id) self.assertEqual(rtls_3.data_state, DATA_STATE_MATCHING) self.assertEqual(rtls_3.merge_state, MERGE_STATE_NEW)
def test_merged_indicators_provided_on_filter_endpoint(self): _import_record, import_file_1 = self.create_import_file( self.user, self.org, self.cycle) base_details = { 'address_line_1': '123 Match Street', 'import_file_id': import_file_1.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } self.taxlot_state_factory.get_taxlot_state(**base_details) # set import_file_1 mapping done so that record is "created for users to view". import_file_1.mapping_done = True import_file_1.save() match_buildings(import_file_1.id) _import_record_2, import_file_2 = self.create_import_file( self.user, self.org, self.cycle) url = reverse( 'api:v2:taxlots-filter' ) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format( self.cycle.pk, self.org.pk) response = self.client.post(url) data = json.loads(response.content) self.assertFalse(data['results'][0]['merged_indicator']) # make sure merged_indicator is True when merge occurs base_details['city'] = 'Denver' base_details['import_file_id'] = import_file_2.id self.taxlot_state_factory.get_taxlot_state(**base_details) # set import_file_2 mapping done so that match merging can occur. import_file_2.mapping_done = True import_file_2.save() match_buildings(import_file_2.id) url = reverse( 'api:v2:taxlots-filter' ) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format( self.cycle.pk, self.org.pk) response = self.client.post(url) data = json.loads(response.content) self.assertTrue(data['results'][0]['merged_indicator']) # Create pairings and check if paired object has indicator as well property_factory = FakePropertyFactory(organization=self.org) property_state_factory = FakePropertyStateFactory( organization=self.org) property = property_factory.get_property() property_state = property_state_factory.get_property_state() property_view = PropertyView.objects.create(property=property, cycle=self.cycle, state=property_state) # attach pairing to one and only taxlot_view TaxLotProperty(primary=True, cycle_id=self.cycle.id, property_view_id=property_view.id, taxlot_view_id=TaxLotView.objects.get().id).save() url = reverse( 'api:v2:taxlots-filter' ) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format( self.cycle.pk, self.org.pk) response = self.client.post(url) data = json.loads(response.content) related = data['results'][0]['related'][0] self.assertTrue('merged_indicator' in related) self.assertFalse(related['merged_indicator'])
def test_match_taxlots_if_all_default_fields_match(self): base_details = { 'address_line_1': '123 Match Street', 'import_file_id': self.import_file.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create first set of taxlots that match each other tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_details) base_details['city'] = 'Denver' tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_details) # Create second set of taxlots that match each other base_details['jurisdiction_tax_lot_id'] = '11111' tls_3 = self.taxlot_state_factory.get_taxlot_state(**base_details) base_details['city'] = 'Philadelphia' tls_4 = self.taxlot_state_factory.get_taxlot_state(**base_details) # Create unmatched taxlot base_details['jurisdiction_tax_lot_id'] = '000' tls_5 = self.taxlot_state_factory.get_taxlot_state(**base_details) # set import_file mapping done so that matching can occur. self.import_file.mapping_done = True self.import_file.save() match_buildings(self.import_file.id) # 3 TaxLot, 3 TaxLotViews, 7 TaxLotStates (5 imported, 2 merge results) self.assertEqual(TaxLot.objects.count(), 3) self.assertEqual(TaxLotView.objects.count(), 3) self.assertEqual(TaxLotState.objects.count(), 7) # Refresh -States and check data_state and merge_state values rtls_1 = TaxLotState.objects.get(pk=tls_1.id) self.assertEqual(rtls_1.data_state, DATA_STATE_MAPPING) self.assertEqual(rtls_1.merge_state, MERGE_STATE_UNKNOWN) rtls_2 = TaxLotState.objects.get(pk=tls_2.id) self.assertEqual(rtls_2.data_state, DATA_STATE_MAPPING) self.assertEqual(rtls_2.merge_state, MERGE_STATE_UNKNOWN) tls_1_plus_2 = TaxLotState.objects.filter( jurisdiction_tax_lot_id__isnull=True, city='Denver', address_line_1='123 Match Street').exclude( data_state=DATA_STATE_MAPPING, merge_state=MERGE_STATE_UNKNOWN).get() self.assertEqual(tls_1_plus_2.data_state, DATA_STATE_MATCHING) self.assertEqual(tls_1_plus_2.merge_state, MERGE_STATE_MERGED) rtls_3 = TaxLotState.objects.get(pk=tls_3.id) self.assertEqual(rtls_3.data_state, DATA_STATE_MAPPING) self.assertEqual(rtls_3.merge_state, MERGE_STATE_UNKNOWN) rtls_4 = TaxLotState.objects.get(pk=tls_4.id) self.assertEqual(rtls_4.data_state, DATA_STATE_MAPPING) self.assertEqual(rtls_4.merge_state, MERGE_STATE_UNKNOWN) tls_3_plus_4 = TaxLotState.objects.filter( jurisdiction_tax_lot_id='11111', city='Philadelphia', address_line_1='123 Match Street').exclude( data_state=DATA_STATE_MAPPING, merge_state=MERGE_STATE_UNKNOWN).get() self.assertEqual(tls_3_plus_4.data_state, DATA_STATE_MATCHING) self.assertEqual(tls_3_plus_4.merge_state, MERGE_STATE_MERGED) rtls_5 = TaxLotState.objects.get(pk=tls_5.id) self.assertEqual(rtls_5.data_state, DATA_STATE_MATCHING) self.assertEqual(rtls_5.merge_state, MERGE_STATE_NEW)
def test_match_properties_if_all_default_fields_match(self): base_details = { 'address_line_1': '123 Match Street', 'import_file_id': self.import_file.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } # Create first set of properties that match each other ps_1 = self.property_state_factory.get_property_state(**base_details) base_details['city'] = 'Denver' ps_2 = self.property_state_factory.get_property_state(**base_details) # Create second set of properties that match each other base_details['pm_property_id'] = '11111' ps_3 = self.property_state_factory.get_property_state(**base_details) base_details['city'] = 'Philadelphia' ps_4 = self.property_state_factory.get_property_state(**base_details) # Create unmatched property base_details['pm_property_id'] = '000' ps_5 = self.property_state_factory.get_property_state(**base_details) # set import_file mapping done so that matching can occur. self.import_file.mapping_done = True self.import_file.save() match_buildings(self.import_file.id) # 3 Property, 3 PropertyViews, 7 PropertyStates (5 imported, 2 merge results) self.assertEqual(Property.objects.count(), 3) self.assertEqual(PropertyView.objects.count(), 3) self.assertEqual(PropertyState.objects.count(), 7) # Refresh -States and check data_state and merge_state values rps_1 = PropertyState.objects.get(pk=ps_1.id) self.assertEqual(rps_1.data_state, DATA_STATE_MAPPING) self.assertEqual(rps_1.merge_state, MERGE_STATE_UNKNOWN) rps_2 = PropertyState.objects.get(pk=ps_2.id) self.assertEqual(rps_2.data_state, DATA_STATE_MAPPING) self.assertEqual(rps_2.merge_state, MERGE_STATE_UNKNOWN) ps_1_plus_2 = PropertyState.objects.filter( pm_property_id__isnull=True, city='Denver', address_line_1='123 Match Street').exclude( data_state=DATA_STATE_MAPPING, merge_state=MERGE_STATE_UNKNOWN).get() self.assertEqual(ps_1_plus_2.data_state, DATA_STATE_MATCHING) self.assertEqual(ps_1_plus_2.merge_state, MERGE_STATE_MERGED) rps_3 = PropertyState.objects.get(pk=ps_3.id) self.assertEqual(rps_3.data_state, DATA_STATE_MAPPING) self.assertEqual(rps_3.merge_state, MERGE_STATE_UNKNOWN) rps_4 = PropertyState.objects.get(pk=ps_4.id) self.assertEqual(rps_4.data_state, DATA_STATE_MAPPING) self.assertEqual(rps_4.merge_state, MERGE_STATE_UNKNOWN) ps_3_plus_4 = PropertyState.objects.filter( pm_property_id='11111', city='Philadelphia', address_line_1='123 Match Street').exclude( data_state=DATA_STATE_MAPPING, merge_state=MERGE_STATE_UNKNOWN).get() self.assertEqual(ps_3_plus_4.data_state, DATA_STATE_MATCHING) self.assertEqual(ps_3_plus_4.merge_state, MERGE_STATE_MERGED) rps_5 = PropertyState.objects.get(pk=ps_5.id) self.assertEqual(rps_5.data_state, DATA_STATE_MATCHING) self.assertEqual(rps_5.merge_state, MERGE_STATE_NEW)
def test_whole_org_match_merge_link_preview_endpoint_taxlots(self): # Cycle 1 / ImportFile 1 - Create 1 taxlot base_taxlot_details = { 'jurisdiction_tax_lot_id': '1st Non-Match Set', 'city': 'City 1', 'district': 'Match Set', 'import_file_id': self.import_file_1.id, 'data_state': DATA_STATE_MAPPING, 'no_default_data': False, } tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_taxlot_details) self.import_file_1.mapping_done = True self.import_file_1.save() match_buildings(self.import_file_1.id) # Cycle 2 / ImportFile 2 - Create 1 unlinked taxlot base_taxlot_details['jurisdiction_tax_lot_id'] = '2nd Non-Match Set' base_taxlot_details['district'] = 'Match Set' base_taxlot_details['import_file_id'] = self.import_file_2.id tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_taxlot_details) self.import_file_2.mapping_done = True self.import_file_2.save() match_buildings(self.import_file_2.id) # Check there doesn't exist links self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id, tls_2.taxlotview_set.first().taxlot_id) url = reverse('api:v3:organizations-match-merge-link-preview', args=[self.org.id]) post_params = json.dumps({ "inventory_type": "taxlots", "add": ['district'], "remove": ['jurisdiction_tax_lot_id'] }) raw_result = self.client.post(url, post_params, content_type='application/json') # Check there *still* doesn't exist links self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id, tls_2.taxlotview_set.first().taxlot_id) self.assertEqual(200, raw_result.status_code) raw_content = json.loads(raw_result.content) identifier = ProgressData.from_key(raw_content['progress_key']).data['unique_id'] result_key = "org_match_merge_link_result__%s" % identifier raw_summary = get_cache_raw(result_key) summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles # Check format of summary self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys()) # Check that preview shows links would be created self.assertEqual(summary[str(self.cycle_1.id)][0]['id'], summary[str(self.cycle_2.id)][0]['id']) # try to get result using results endpoint get_result_url = reverse('api:v3:organizations-match-merge-link-result', args=[self.org.id]) + '?match_merge_link_id=' + str(identifier) get_result_raw_response = self.client.get(get_result_url) raw_summary = json.loads(get_result_raw_response.content) summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles # Check format of summary self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys()) # Check that preview shows links would be created self.assertEqual(summary[str(self.cycle_1.id)][0]['id'], summary[str(self.cycle_2.id)][0]['id'])
def test_demo_v2(self): tasks._save_raw_data(self.import_file_tax_lot.pk, 'fake_cache_key', 1) Column.create_mappings(self.fake_taxlot_mappings, self.org, self.user) Column.create_mappings(self.fake_portfolio_mappings, self.org, self.user) tasks.map_data(self.import_file_tax_lot.pk) # Check to make sure the taxlots were imported ts = TaxLotState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_tax_lot, ) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) self.assertEqual(len(ps), 0) self.assertEqual(len(ts), 9) tasks.match_buildings(self.import_file_tax_lot.id, self.user.id) # Check a single case of the taxlotstate self.assertEqual(TaxLotState.objects.filter(address_line_1='050 Willow Ave SE').count(), 1) self.assertEqual( TaxLotView.objects.filter(state__address_line_1='050 Willow Ave SE').count(), 1 ) self.assertEqual(TaxLotView.objects.count(), 9) # Import the property data tasks._save_raw_data(self.import_file_property.pk, 'fake_cache_key', 1) tasks.map_data(self.import_file_property.pk) ts = TaxLotState.objects.filter( # data_state=DATA_STATE_MAPPING, # Look at all taxlotstates organization=self.org, import_file=self.import_file_tax_lot, ) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) self.assertEqual(len(ts), 9) self.assertEqual(len(ps), 14) tasks.match_buildings(self.import_file_property.id, self.user.id) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) # there shouldn't be any properties left in the mapping state self.assertEqual(len(ps), 0) # psv = PropertyView.objects.filter(state__organization=self.org) # self.assertEqual(len(psv), 12) # tlv = TaxLotView.objects.filter(state__organization=self.org) # self.assertEqual(len(tlv), 9) self.assertEqual(PropertyView.objects.filter(state__organization=self.org, state__pm_property_id='2264').count(), 1) pv = PropertyView.objects.filter(state__organization=self.org, state__pm_property_id='2264').first() self.assertEqual(pv.state.property_name, 'University Inn') self.assertEqual(pv.state.address_line_1, '50 Willow Ave SE')
def test_check_multiple_text_match(self): d = DataQualityCheck.retrieve(self.org) d.remove_all_rules() sl_data = {'name': 'No meters present', 'super_organization': self.org} sl_ok_1, _ = StatusLabel.objects.get_or_create(**sl_data) new_rule = { 'table_name': 'PropertyState', 'field': 'meters_present', 'data_type': TYPE_STRING, 'rule_type': RULE_TYPE_CUSTOM, 'text_match': 'OK', 'severity': SEVERITY_ERROR, 'status_label': sl_ok_1, } d.add_rule(new_rule) sl_data = { 'name': 'No 12 Consectutive Months', 'super_organization': self.org } sl_ok_2, _ = StatusLabel.objects.get_or_create(**sl_data) new_rule = { 'table_name': 'PropertyState', 'field': '12 Consectutive Months', 'data_type': TYPE_STRING, 'rule_type': RULE_TYPE_CUSTOM, 'text_match': 'OK', 'severity': SEVERITY_ERROR, 'status_label': sl_ok_2, } d.add_rule(new_rule) sl_data = {'name': 'No Monthly Data', 'super_organization': self.org} sl_ok_3, _ = StatusLabel.objects.get_or_create(**sl_data) new_rule = { 'table_name': 'PropertyState', 'field': 'Monthly Data', 'data_type': TYPE_STRING, 'rule_type': RULE_TYPE_CUSTOM, 'text_match': 'OK', 'severity': SEVERITY_ERROR, 'status_label': sl_ok_3, } d.add_rule(new_rule) # import data tasks.save_raw_data(self.import_file.id) Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk) tasks.map_data(self.import_file.id) tasks.match_buildings(self.import_file.id) qs = PropertyState.objects.filter( import_file=self.import_file, source_type=ASSESSED_BS, ).iterator() d.reset_results() d.check_data('PropertyState', qs) # Check multiple strings props = PropertyView.objects.filter( property__labels=sl_ok_1).select_related('state') addresses = sorted([p.state.address_line_1 for p in props]) expected = [ u'1 International Road', u'17246 Esch Drive', u'2581 Schiller Parkway', u'3 Northport Place', u'84807 Buell Trail' ] self.assertListEqual(expected, addresses) props = PropertyView.objects.filter( property__labels=sl_ok_2).select_related('state') addresses = sorted([p.state.address_line_1 for p in props]) expected = [ u'1 International Road', u'2581 Schiller Parkway', u'49705 Harper Crossing' ] self.assertListEqual(expected, addresses) props = PropertyView.objects.filter( property__labels=sl_ok_3).select_related('state') addresses = sorted([p.state.address_line_1 for p in props]) expected = [ u'1 International Road', u'17246 Esch Drive', u'84807 Buell Trail', u'88263 Scoville Park' ] self.assertListEqual(expected, addresses)
def test_delete_organization_buildings(self): """tests the delete buildings for an organization""" # start with the normal use case bs1_data = { 'pm_property_id': 123, # 'tax_lot_id': '435/422', 'property_name': 'Greenfield Complex', 'custom_id_1': 1243, 'address_line_1': '555 NorthWest Databaseer Lane.', 'address_line_2': '', 'city': 'Gotham City', 'postal_code': 8999, } # This building will have a lot less data to identify it. bs2_data = { 'pm_property_id': 1243, 'custom_id_1': 1243, 'address_line_1': '555 Database LN.', 'city': 'Gotham City', 'postal_code': 8999, } new_import_file = ImportFile.objects.create( import_record=self.import_record, mapping_done=True ) snapshot = util.make_fake_property( self.import_file, bs1_data, ASSESSED_BS, is_canon=True ) snapshot.super_organization = self.fake_org snapshot.save() snapshot = util.make_fake_property( new_import_file, bs2_data, PORTFOLIO_BS ) snapshot.super_organization = self.fake_org snapshot.save() # TODO: do not import this data to test the deleting di_tasks.match_buildings(new_import_file.pk, self.fake_user.pk) # make one more building snapshot in a different org fake_org_2 = Organization.objects.create() snapshot = util.make_fake_property( self.import_file, bs1_data, ASSESSED_BS, is_canon=True ) snapshot.super_organization = fake_org_2 snapshot.save() self.assertGreater(BuildingSnapshot.objects.filter( super_organization=self.fake_org ).count(), 0) tasks.delete_organization_buildings(self.fake_org.pk, 'fake-progress-key') self.assertEqual(BuildingSnapshot.objects.filter( super_organization=self.fake_org ).count(), 0) self.assertGreater(BuildingSnapshot.objects.filter( super_organization=fake_org_2 ).count(), 0) # test that the CanonicalBuildings are deleted self.assertEqual(CanonicalBuilding.objects.filter( canonical_snapshot__super_organization=self.fake_org ).count(), 0) # test that other orgs CanonicalBuildings are not deleted self.assertGreater(CanonicalBuilding.objects.filter( canonical_snapshot__super_organization=fake_org_2 ).count(), 0)
def test_demo_v2(self): tasks.save_raw_data(self.import_file_tax_lot.pk) Column.create_mappings(self.fake_taxlot_mappings, self.org, self.user) Column.create_mappings(self.fake_portfolio_mappings, self.org, self.user) tasks.map_data(self.import_file_tax_lot.pk) # Check to make sure the taxlots were imported ts = TaxLotState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_tax_lot, ) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) self.assertEqual(len(ps), 0) self.assertEqual(len(ts), 9) tasks.match_buildings(self.import_file_tax_lot.id) # Check a single case of the taxlotstate self.assertEqual(TaxLotState.objects.filter(address_line_1='2655 Welstone Ave NE').count(), 1) self.assertEqual( TaxLotView.objects.filter(state__address_line_1='2655 Welstone Ave NE').count(), 1 ) self.assertEqual(TaxLotView.objects.count(), 9) # Import the property data tasks.save_raw_data(self.import_file_property.pk) tasks.map_data(self.import_file_property.pk) ts = TaxLotState.objects.filter( # data_state=DATA_STATE_MAPPING, # Look at all taxlotstates organization=self.org, import_file=self.import_file_tax_lot, ) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) self.assertEqual(len(ts), 9) self.assertEqual(len(ps), 14) tasks.match_buildings(self.import_file_property.id) ps = PropertyState.objects.filter( data_state=DATA_STATE_MAPPING, organization=self.org, import_file=self.import_file_property, ) # there should not be any properties left in the mapping state self.assertEqual(len(ps), 0) # psv = PropertyView.objects.filter(state__organization=self.org) # self.assertEqual(len(psv), 12) # tlv = TaxLotView.objects.filter(state__organization=self.org) # self.assertEqual(len(tlv), 9) self.assertEqual(PropertyView.objects.filter(state__organization=self.org, state__pm_property_id='2264').count(), 1) pv = PropertyView.objects.filter(state__organization=self.org, state__pm_property_id='2264').first() self.assertEqual(pv.state.property_name, 'University Inn') self.assertEqual(pv.state.address_line_1, '50 Willow Ave SE')