def setUp(self):
# for now just import some test data. I'd rather create fake data... next time.
filename = getattr(self, 'filename', 'example-data-properties.xlsx')
self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio'])
selfvars = self.set_up(ASSESSED_RAW)
self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars
filepath = osp.join(osp.dirname(__file__), '..', 'data', filename)
self.import_file.file = SimpleUploadedFile(name=filename,
content=open(
filepath, 'rb').read())
self.import_file.save()
tasks.save_raw_data(self.import_file.pk)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.id)
tasks.map_data(self.import_file.pk)
tasks.match_buildings(self.import_file.id)
# import second file with tax lot information
filename_2 = getattr(self, 'filename', 'example-data-taxlots.xlsx')
self.fake_mappings = copy.copy(FAKE_MAPPINGS['taxlot'])
_, self.import_file_2 = self.create_import_file(
self.user, self.org, self.cycle)
filepath = osp.join(osp.dirname(__file__), '..', 'data', filename_2)
self.import_file_2.file = SimpleUploadedFile(name=filename_2,
content=open(
filepath,
'rb').read())
self.import_file_2.save()
tasks.save_raw_data(self.import_file_2.pk)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.id)
tasks.map_data(self.import_file_2.pk)
tasks.match_buildings(self.import_file_2.id)
def test_no_matches_if_all_matching_criteria_is_None(self):
"""
Default matching criteria for PropertyStates are:
- address_line_1 (substituted by normalized_address)
- ubid
- pm_property_id
- custom_id_1
and all are set to None.
"""
base_details = {
'import_file_id': self.import_file.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create set of properties that won't match
self.property_state_factory.get_property_state(**base_details)
base_details['city'] = 'Denver'
self.property_state_factory.get_property_state(**base_details)
# set import_file mapping done so that matching can occur.
self.import_file.mapping_done = True
self.import_file.save()
match_buildings(self.import_file.id)
# 2 Property, 2 PropertyView, 2 PropertyStates - No merges
self.assertEqual(Property.objects.count(), 2)
self.assertEqual(PropertyView.objects.count(), 2)
self.assertEqual(PropertyState.objects.count(), 2)
def test_duplicate_properties_identified(self):
base_details = {
'address_line_1': '123 Match Street',
'import_file_id': self.import_file_1.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create property in first ImportFile
ps_1 = self.property_state_factory.get_property_state(**base_details)
self.import_file_1.mapping_done = True
self.import_file_1.save()
match_buildings(self.import_file_1.id)
# Create duplicate property coming from second ImportFile
base_details['import_file_id'] = self.import_file_2.id
ps_2 = self.property_state_factory.get_property_state(**base_details)
self.import_file_2.mapping_done = True
self.import_file_2.save()
match_buildings(self.import_file_2.id)
# 1 Property, 1 PropertyViews, 2 PropertyStates
self.assertEqual(Property.objects.count(), 1)
self.assertEqual(PropertyView.objects.count(), 1)
self.assertEqual(PropertyState.objects.count(), 2)
# Be sure the first property is used in the -View and the second is marked for "deletion"
self.assertEqual(PropertyView.objects.first().state_id, ps_1.id)
self.assertEqual(
PropertyState.objects.get(data_state=DATA_STATE_DELETE).id,
ps_2.id)
def setUp(self):
super(TestProperties, self).setUp()
# for now just import some test data. I'd rather create fake data... next time.
filename = getattr(self, 'filename', 'example-data-properties.xlsx')
self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio'])
selfvars = self.set_up(ASSESSED_RAW)
self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars
filepath = osp.join(osp.dirname(__file__), '..', 'data', filename)
self.import_file.file = SimpleUploadedFile(name=filename,
content=open(
filepath, 'rb').read())
self.import_file.save()
tasks.save_raw_data(self.import_file.pk)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.id)
tasks.map_data(self.import_file.pk)
tasks.match_buildings(self.import_file.id)
# import second file that is currently the same, but should be slightly different
filename_2 = getattr(self, 'filename',
'example-data-properties-small-changes.xlsx')
_, self.import_file_2 = self.create_import_file(
self.user, self.org, self.cycle)
filepath = osp.join(osp.dirname(__file__), '..', 'data', filename_2)
self.import_file_2.file = SimpleUploadedFile(name=filename_2,
content=open(
filepath,
'rb').read())
self.import_file_2.save()
tasks.save_raw_data(self.import_file_2.pk)
tasks.map_data(self.import_file_2.pk)
tasks.match_buildings(self.import_file_2.id)
def test_match_properties_get_rolled_up_into_one_in_the_order_their_uploaded(self):
"""
The most recently uploaded should take precedence when merging states.
If more than 2 states match each other, they are merged two at a time
until one is remaining.
Reminder, this is only for -States within an ImportFile.
"""
base_details = {
'address_line_1': '123 Match Street',
'import_file_id': self.import_file.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create first set of properties that match each other
base_details['city'] = 'Philadelphia'
self.property_state_factory.get_property_state(**base_details)
base_details['city'] = 'Arvada'
self.property_state_factory.get_property_state(**base_details)
base_details['city'] = 'Golden'
self.property_state_factory.get_property_state(**base_details)
base_details['city'] = 'Denver'
self.property_state_factory.get_property_state(**base_details)
# set import_file mapping done so that matching can occur.
self.import_file.mapping_done = True
self.import_file.save()
match_buildings(self.import_file.id)
# 1 Property, 1 PropertyViews, 7 PropertyStates (4 imported, 3 merge results)
self.assertEqual(Property.objects.count(), 1)
self.assertEqual(PropertyView.objects.count(), 1)
self.assertEqual(PropertyState.objects.count(), 7)
self.assertEqual(PropertyView.objects.first().state.city, 'Denver')
def test_duplicate_taxlots_identified(self):
base_details = {
'address_line_1': '123 Match Street',
'import_file_id': self.import_file.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create pair of properties that are exact duplicates
self.taxlot_state_factory.get_taxlot_state(**base_details)
self.taxlot_state_factory.get_taxlot_state(**base_details)
# Create a non-matching, non-duplicate property
base_details['address_line_1'] = '123 Different Ave'
base_details['city'] = 'Denver'
self.taxlot_state_factory.get_taxlot_state(**base_details)
# set import_file mapping done so that matching can occur.
self.import_file.mapping_done = True
self.import_file.save()
match_buildings(self.import_file.id)
# 2 TaxLot, 2 TaxLotViews, 3 TaxLotState (1 flagged to be ignored)
self.assertEqual(TaxLot.objects.count(), 2)
self.assertEqual(TaxLotView.objects.count(), 2)
self.assertEqual(TaxLotState.objects.count(), 3)
self.assertEqual(
TaxLotState.objects.filter(data_state=DATA_STATE_DELETE).count(),
1)
# Make sure "deleted" -States are not found in the -Views
deleted = TaxLotState.objects.get(data_state=DATA_STATE_DELETE)
self.assertNotIn(deleted.id,
TaxLotView.objects.values_list('state_id', flat=True))
def test_import_duplicates(self):
# Check to make sure all the properties imported
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file,
)
self.assertEqual(len(ps), 9)
self.assertEqual(PropertyState.objects.filter(pm_property_id='2264').count(), 7)
hashes = list(map(tasks.hash_state_object, ps))
self.assertEqual(len(hashes), 9)
self.assertEqual(len(set(hashes)), 4)
unique_property_states, _ = match.filter_duplicate_states(ps)
self.assertEqual(len(unique_property_states), 4)
tasks.match_buildings(self.import_file.id)
self.assertEqual(Property.objects.count(), 3)
self.assertEqual(PropertyView.objects.count(), 3)
self.assertEqual(PropertyView.objects.filter(state__pm_property_id='2264').count(), 1)
pv = PropertyView.objects.filter(state__pm_property_id='2264').first()
self.assertEqual(pv.state.pm_property_id, '2264')
self.assertEqual(pv.state.gross_floor_area, 12555 * ureg.feet ** 2)
self.assertEqual(pv.state.energy_score, 75)
self.assertEqual(TaxLot.objects.count(), 0)
self.assertEqual(self.import_file.find_unmatched_property_states().count(), 2)
self.assertEqual(self.import_file.find_unmatched_tax_lot_states().count(), 0)
def test_match_merge_in_cycle_ignores_taxlots_with_unpopulated_matching_criteria(
self):
base_details = {
'city': 'Golden',
'import_file_id': self.import_file_1.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create 3 non-duplicate taxlots with unpopulated matching criteria
tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_details)
base_details['city'] = 'Denver'
tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_details)
base_details['city'] = 'Philadelphia'
tls_3 = self.taxlot_state_factory.get_taxlot_state(**base_details)
self.import_file_1.mapping_done = True
self.import_file_1.save()
match_buildings(self.import_file_1.id)
# Verify no match merges happen
tls_1_view = TaxLotView.objects.get(state_id=tls_1.id)
count_result, no_match_indicator = match_merge_in_cycle(
tls_1_view.id, 'TaxLotState')
self.assertEqual(count_result, 0)
self.assertIsNone(no_match_indicator)
self.assertEqual(TaxLot.objects.count(), 3)
self.assertEqual(TaxLotState.objects.count(), 3)
self.assertEqual(TaxLotView.objects.count(), 3)
state_ids = list(TaxLotView.objects.all().values_list('state_id',
flat=True))
self.assertCountEqual([tls_1.id, tls_2.id, tls_3.id], state_ids)
def setUp(self):
filename = getattr(self, 'filename', 'example-data-properties.xlsx')
self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio'])
selfvars = self.set_up(ASSESSED_RAW)
self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars
self.import_file.load_import_file(
osp.join(osp.dirname(__file__), 'data', filename))
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.id)
tasks.map_data(self.import_file.pk)
tasks.match_buildings(self.import_file.id)
# import second file that is currently the same, but should be slightly different
filename_2 = getattr(self, 'filename',
'example-data-properties-small-changes.xlsx')
_, self.import_file_2 = self.create_import_file(
self.user, self.org, self.cycle)
self.import_file_2.load_import_file(
osp.join(osp.dirname(__file__), 'data', filename_2))
tasks._save_raw_data(self.import_file_2.pk, 'fake_cache_key_2', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file_2.id)
tasks.map_data(self.import_file_2.pk)
tasks.match_buildings(self.import_file_2.id)
# for api tests
user_details = {
'username': '******',
'password': '******',
}
self.client.login(**user_details)
def setUp(self):
# for now just import some test data. I'd rather create fake data... next time.
filename = getattr(self, 'filename', 'example-data-properties.xlsx')
self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio'])
selfvars = self.set_up(ASSESSED_RAW)
self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars
self.import_file.load_import_file(
osp.join(osp.dirname(__file__), '../data_importer/tests/data',
filename))
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.id)
tasks.map_data(self.import_file.pk)
tasks.match_buildings(self.import_file.id)
# import second file that is currently the same, but should be slightly different
filename_2 = getattr(self, 'filename',
'example-data-properties-small-changes.xlsx')
_, self.import_file_2 = self.create_import_file(
self.user, self.org, self.cycle)
self.import_file_2.load_import_file(
osp.join(osp.dirname(__file__), '../data_importer/tests/data',
filename_2))
tasks._save_raw_data(self.import_file_2.pk, 'fake_cache_key_2', 1)
tasks.map_data(self.import_file_2.pk)
tasks.match_buildings(self.import_file_2.id)
def test_match_properties_rolls_up_multiple_existing_matches_in_id_order_if_they_exist(self):
base_details = {
'pm_property_id': '123MatchID',
'city': 'Golden',
'import_file_id': self.import_file_1.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create 3 non-matching properties in first ImportFile
ps_1 = self.property_state_factory.get_property_state(**base_details)
base_details['pm_property_id'] = '789DifferentID'
base_details['city'] = 'Denver'
ps_2 = self.property_state_factory.get_property_state(**base_details)
base_details['pm_property_id'] = '1337AnotherDifferentID'
base_details['city'] = 'Philadelphia'
ps_3 = self.property_state_factory.get_property_state(**base_details)
self.import_file_1.mapping_done = True
self.import_file_1.save()
match_buildings(self.import_file_1.id)
# Update -States to make the roll up order be 1, 3, 2
refreshed_ps_3 = PropertyState.objects.get(id=ps_3.id)
refreshed_ps_3.pm_property_id = '123MatchID'
refreshed_ps_3.save()
refreshed_ps_2 = PropertyState.objects.get(id=ps_2.id)
refreshed_ps_2.pm_property_id = '123MatchID'
refreshed_ps_2.save()
# Verify that none of the 3 have been merged
self.assertEqual(Property.objects.count(), 3)
self.assertEqual(PropertyState.objects.count(), 3)
self.assertEqual(PropertyView.objects.count(), 3)
# Import a property that will identify the first 3 as matches.
base_details['import_file_id'] = self.import_file_2.id
base_details['pm_property_id'] = '123MatchID'
del base_details['city']
ps_4 = self.property_state_factory.get_property_state(**base_details)
self.import_file_2.mapping_done = True
self.import_file_2.save()
match_buildings(self.import_file_2.id)
# There should only be one PropertyView which is associated to new, merged -State
self.assertEqual(PropertyView.objects.count(), 1)
view = PropertyView.objects.first()
self.assertNotIn(view.state_id, [ps_1.id, ps_2.id, ps_3.id, ps_4.id])
# It will have a -State having city as Denver
self.assertEqual(view.state.city, 'Denver')
# The corresponding log should be a System Match
audit_log = PropertyAuditLog.objects.get(state_id=view.state_id)
self.assertEqual(audit_log.name, 'System Match')
def test_match_properties_and_taxlots_with_ubid(self):
# create an ImportFile for testing purposes. Seems like we would want to run this matching just on a
# list of properties and taxlots.
#
# This emulates importing the following
# UBID, Jurisdiction Tax Lot
# ubid_100, lot_1
# ubid_101, lot_1
# ubid_110, lot_1
# ubid_111, lot_1
ids = [('ubid_100', 'lot_1'), ('ubid_101', 'lot_1'),
('ubid_110', 'lot_1'), ('ubid_111', 'lot_1')]
for id in ids:
self.property_state_factory.get_property_state(
no_default_data=True,
ubid=id[0],
lot_number=id[1],
import_file_id=self.import_file.id,
data_state=DATA_STATE_MAPPING,
)
self.taxlot_state_factory.get_taxlot_state(
no_default_data=True,
jurisdiction_tax_lot_id=ids[0][1],
import_file_id=self.import_file.id,
data_state=DATA_STATE_MAPPING,
)
# for ps in PropertyState.objects.filter(organization=self.org):
# print("%s -- %s -- %s" % (ps.lot_number, ps.import_file_id, ps.ubid))
# pv = PropertyView.objects.get(state=ps, cycle=self.cycle)
# TaxLotProperty.objects.filter()
# for tl in TaxLotState.objects.filter(organization=self.org):
# print("%s -- %s" % (tl.import_file_id, tl.jurisdiction_tax_lot_id))
# set import_file mapping done so that matching can occur.
self.import_file.mapping_done = True
self.import_file.save()
match_buildings(self.import_file.id)
# for pv in PropertyView.objects.filter(state__organization=self.org):
# print("%s -- %s" % (pv.state.ubid, pv.cycle))
# should only have 1 PropertyView and 4 taxlot views
self.assertEqual(
PropertyView.objects.filter(state__organization=self.org).count(),
4)
self.assertEqual(
TaxLotView.objects.filter(state__organization=self.org).count(), 1)
tlv = TaxLotView.objects.filter(state__organization=self.org).first()
# there should be 4 relationships in the TaxLotProperty associated with view, one each for the taxlots defined
self.assertEqual(
TaxLotProperty.objects.filter(taxlot_view_id=tlv).count(), 4)
def test_match_taxlots_rolls_up_multiple_existing_matches_in_id_order_if_they_exist(
self):
base_details = {
'jurisdiction_tax_lot_id': '123MatchID',
'city': 'Golden',
'import_file_id': self.import_file_1.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create 3 non-matching taxlots in first ImportFile
tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_details)
base_details['jurisdiction_tax_lot_id'] = '789DifferentID'
base_details['city'] = 'Denver'
tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_details)
base_details['jurisdiction_tax_lot_id'] = '1337AnotherDifferentID'
base_details['city'] = 'Philadelphia'
tls_3 = self.taxlot_state_factory.get_taxlot_state(**base_details)
self.import_file_1.mapping_done = True
self.import_file_1.save()
match_buildings(self.import_file_1.id)
# Make all those states match
TaxLotState.objects.filter(pk__in=[tls_2.id, tls_3.id]).update(
jurisdiction_tax_lot_id='123MatchID')
# Verify that none of the 3 have been merged
self.assertEqual(TaxLot.objects.count(), 3)
self.assertEqual(TaxLotState.objects.count(), 3)
self.assertEqual(TaxLotView.objects.count(), 3)
# Import a property that will identify the first 3 as matches.
base_details['import_file_id'] = self.import_file_2.id
base_details['jurisdiction_tax_lot_id'] = '123MatchID'
del base_details['city']
tls_4 = self.taxlot_state_factory.get_taxlot_state(**base_details)
self.import_file_2.mapping_done = True
self.import_file_2.save()
match_buildings(self.import_file_2.id)
# There should only be one TaxLotView which is associated to new, merged -State
self.assertEqual(TaxLotView.objects.count(), 1)
view = TaxLotView.objects.first()
self.assertNotIn(view.state_id,
[tls_1.id, tls_2.id, tls_3.id, tls_4.id])
# It will have a -State having city as Philadelphia
self.assertEqual(view.state.city, 'Philadelphia')
# The corresponding log should be a System Match
audit_log = TaxLotAuditLog.objects.get(state_id=view.state_id)
self.assertEqual(audit_log.name, 'System Match')
def test_match_buildings(self):
""" case A (one property <-> one tax lot) """
tasks.save_raw_data(self.import_file.pk)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.pk)
tasks.map_data(self.import_file.pk)
# Check to make sure all the properties imported
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file,
)
self.assertEqual(len(ps), 14)
# Check to make sure the taxlots were imported
ts = TaxLotState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file,
)
self.assertEqual(len(ts), 18)
# Check a single case of the taxlotstate
ts = TaxLotState.objects.filter(
jurisdiction_tax_lot_id='1552813').first()
self.assertEqual(ts.jurisdiction_tax_lot_id, '1552813')
self.assertEqual(ts.address_line_1, None)
self.assertEqual(ts.extra_data["data_008"], 1)
# Check a single case of the propertystate
ps = PropertyState.objects.filter(pm_property_id='2264')
self.assertEqual(len(ps), 1)
ps = ps.first()
self.assertEqual(ps.pm_property_id, '2264')
self.assertEqual(ps.address_line_1, '50 Willow Ave SE')
self.assertEqual('data_007' in ps.extra_data, True)
self.assertEqual('data_008' in ps.extra_data, False)
self.assertEqual(ps.extra_data["data_007"], 'a')
# verify that the lot_number has the tax_lot information. For this case it is one-to-one
self.assertEqual(ps.lot_number, ts.jurisdiction_tax_lot_id)
tasks.match_buildings(self.import_file.id)
self.assertEqual(TaxLot.objects.count(), 10)
qry = PropertyView.objects.filter(state__custom_id_1='7')
self.assertEqual(qry.count(), 1)
state = qry.first().state
self.assertEqual(state.address_line_1, "12 Ninth Street")
self.assertEqual(state.property_name, "Grange Hall")
def test_match_properties_and_taxlots_with_address_no_lot_number(self):
# create an ImportFile for testing purposes. Seems like we would want to run this matching just on a
# list of properties and taxlots.
#
# This emulates importing the following
# Address, Jurisdiction Tax Lot
# 742 Evergreen Terrace, 100
# 742 Evergreen Terrace, 101
# 742 Evergreen Terrace, 110
# 742 Evergreen Terrace, 111
lot_numbers = '100;101;110;111'
for lot_number in lot_numbers.split(';'):
self.property_state_factory.get_property_state(
address_line_1='742 Evergreen Terrace',
lot_number=lot_number,
import_file_id=self.import_file.id,
data_state=DATA_STATE_MAPPING,
)
self.taxlot_state_factory.get_taxlot_state(
address_line_1=None,
jurisdiction_tax_lot_id=lot_number,
import_file_id=self.import_file.id,
data_state=DATA_STATE_MAPPING,
)
# for ps in PropertyState.objects.filter(organization=self.org):
# print("%s -- %s -- %s" % (ps.lot_number, ps.import_file_id, ps.address_line_1))
# for tl in TaxLotState.objects.filter(organization=self.org):
# print("%s -- %s" % (tl.import_file_id, tl.jurisdiction_tax_lot_id))
# set import_file mapping done so that matching can occur.
self.import_file.mapping_done = True
self.import_file.save()
match_buildings(self.import_file.id)
# for pv in PropertyView.objects.filter(state__organization=self.org):
# print("%s -- %s" % (pv.state, pv.cycle))
# should only have 1 PropertyView and 4 taxlot views
self.assertEqual(
PropertyView.objects.filter(state__organization=self.org).count(),
1)
self.assertEqual(
TaxLotView.objects.filter(state__organization=self.org).count(), 4)
pv = PropertyView.objects.filter(state__organization=self.org).first()
# there should be 4 relationships in the TaxLotProperty associated with view, one each for the taxlots defined
self.assertEqual(
TaxLotProperty.objects.filter(property_view_id=pv).count(), 4)
def test_match_buildings(self):
""" case A (one property <-> one tax lot) """
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user)
tasks.map_data(self.import_file.pk)
# Check to make sure all the properties imported
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file,
)
self.assertEqual(len(ps), 14)
# Check to make sure the taxlots were imported
ts = TaxLotState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file,
)
# self.assertEqual(len(ts), 10) # 10 unique taxlots after duplicates and delimeters
# Check a single case of the taxlotstate
ts = TaxLotState.objects.filter(jurisdiction_tax_lot_id='1552813').first()
self.assertEqual(ts.jurisdiction_tax_lot_id, '1552813')
self.assertEqual(ts.address_line_1, None)
self.assertEqual(ts.extra_data["data_008"], 1)
# Check a single case of the propertystate
ps = PropertyState.objects.filter(pm_property_id='2264')
self.assertEqual(len(ps), 1)
ps = ps.first()
self.assertEqual(ps.pm_property_id, '2264')
self.assertEqual(ps.address_line_1, '50 Willow Ave SE')
self.assertEqual('data_007' in ps.extra_data.keys(), True)
self.assertEqual('data_008' in ps.extra_data.keys(), False)
self.assertEqual(ps.extra_data["data_007"], 'a')
# verify that the lot_number has the tax_lot information. For this case it is one-to-one
self.assertEqual(ps.lot_number, ts.jurisdiction_tax_lot_id)
tasks.match_buildings(self.import_file.id, self.user.id)
self.assertEqual(TaxLot.objects.count(), 10)
qry = PropertyView.objects.filter(state__custom_id_1='7')
self.assertEqual(qry.count(), 1)
state = qry.first().state
self.assertEqual(state.address_line_1, "12 Ninth Street")
self.assertEqual(state.property_name, "Grange Hall")
def _create_propertyview(request, org, user, dataset_name):
cycle = Cycle.objects.filter(
organization=org).last() # might need to hardcode this
dataset = ImportRecord.objects.get(name=dataset_name,
super_organization=org)
result = [{'City': request.GET['city'], 'State': request.GET['state']}]
if 'street' in request.GET:
result[0]['Address Line 1'] = request.GET['street']
else:
result[0]['Address Line 1'] = request.GET['address_line_1']
if 'zipcode' in request.GET:
result[0]['Postal Code'] = request.GET['zipcode']
else:
result[0]['Postal Code'] = request.GET['postal_code']
if 'property_uid' in request.GET:
result[0]['Custom ID 1'] = request.GET['property_uid']
file_pk = utils.save_and_load(user, dataset, cycle, result,
"profile_data.csv")
# save data
resp = save_raw_data(file_pk)
save_prog_key = resp['progress_key']
utils.wait_for_task(save_prog_key)
# map data
# save_column_mappings(file_id, col_mappings) #perform column mapping
resp = map_data(file_pk)
map_prog_key = resp['progress_key']
utils.wait_for_task(map_prog_key)
resp = match_buildings(file_pk)
# resp = geocode_buildings_task(file_pk)
if (resp['status'] == 'error'):
return resp
match_prog_key = resp['progress_key']
utils.wait_for_task(match_prog_key)
propertyview = utils.propertyview_find(request)
return propertyview
def test_handle_id_matches_duplicate_data(self):
"""
Test for handle_id_matches behavior when matching duplicate data
"""
# TODO: Fix the PM, tax lot id, and custom ID fields in PropertyState
bs_data = {
'pm_property_id': "2360",
# 'tax_lot_id': '476/460',
'property_name': 'Garfield Complex',
'custom_id_1': "89",
'address_line_1': '12975 Database LN.',
'address_line_2': '',
'city': 'Cartoon City',
'postal_code': "54321",
'data_state': DATA_STATE_MAPPING,
'source_type': ASSESSED_BS,
}
# Setup mapped AS snapshot.
PropertyState.objects.create(
organization=self.org,
import_file=self.import_file,
**bs_data
)
# Different file, but same ImportRecord.
# Setup mapped PM snapshot.
# Should be an identical match.
new_import_file = ImportFile.objects.create(import_record=self.import_record,
mapping_done=True)
tasks.match_buildings(new_import_file.pk, self.user.pk)
duplicate_import_file = ImportFile.objects.create(
import_record=self.import_record,
mapping_done=True
)
PropertyState.objects.create(
organization=self.org,
import_file=duplicate_import_file,
**bs_data
)
def test_handle_id_matches_duplicate_data(self):
"""
Test for handle_id_matches behavior when matching duplicate data
"""
# TODO: Fix the PM, tax lot id, and custom ID fields in PropertyState
bs_data = {
'pm_property_id': "2360",
# 'tax_lot_id': '476/460',
'property_name': 'Garfield Complex',
'custom_id_1': "89",
'address_line_1': '12975 Database LN.',
'address_line_2': '',
'city': 'Cartoon City',
'postal_code': "54321",
'data_state': DATA_STATE_MAPPING,
'source_type': ASSESSED_BS,
}
# Setup mapped AS snapshot.
PropertyState.objects.create(
organization=self.org,
import_file=self.import_file,
**bs_data
)
# Different file, but same ImportRecord.
# Setup mapped PM snapshot.
# Should be an identical match.
new_import_file = ImportFile.objects.create(import_record=self.import_record,
mapping_done=True)
tasks.match_buildings(new_import_file.pk)
duplicate_import_file = ImportFile.objects.create(
import_record=self.import_record,
mapping_done=True
)
PropertyState.objects.create(
organization=self.org,
import_file=duplicate_import_file,
**bs_data
)
def test_match_buildings(self):
""" case B (many property <-> one tax lot) """
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.pk)
# Set remap to True because for some reason this file id has been imported before.
tasks.map_data(self.import_file.pk, True)
# Check to make sure all the properties imported
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file,
)
self.assertEqual(len(ps), 14)
# Check to make sure the tax lots were imported
ts = TaxLotState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file,
)
self.assertEqual(len(ts), 18)
# verify that the lot_number has the tax_lot information. For this case it is one-to-many
p_test = PropertyState.objects.filter(
pm_property_id='5233255',
organization=self.org,
data_state=DATA_STATE_MAPPING,
import_file=self.import_file,
).first()
self.assertEqual(p_test.lot_number, "333/66555;333/66125;333/66148")
tasks.match_buildings(self.import_file.id)
# make sure the the property only has one tax lot and vice versa
tlv = TaxLotView.objects.filter(
state__jurisdiction_tax_lot_id='11160509', cycle=self.cycle)
self.assertEqual(len(tlv), 1)
tlv = tlv[0]
properties = tlv.property_states()
self.assertEqual(len(properties), 3)
def setUp(self):
data_importer_data_dir = os.path.join(os.path.dirname(__file__), '..',
'data_importer', 'tests', 'data')
filename = getattr(self, 'filename', 'example-data-properties.xlsx')
self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio'])
selfvars = self.set_up(ASSESSED_RAW)
self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars
filepath = os.path.join(data_importer_data_dir, filename)
self.import_file.file = SimpleUploadedFile(name=filename,
content=open(
filepath, 'rb').read())
self.import_file.save()
tasks.save_raw_data(self.import_file.pk)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.id)
tasks.map_data(self.import_file.pk)
tasks.match_buildings(self.import_file.id)
# import second file that is currently the same, but should be slightly different
filename_2 = getattr(self, 'filename',
'example-data-properties-small-changes.xlsx')
_, self.import_file_2 = self.create_import_file(
self.user, self.org, self.cycle)
filepath = os.path.join(data_importer_data_dir, filename_2)
self.import_file_2.file = SimpleUploadedFile(name=filename_2,
content=open(
filepath,
'rb').read())
self.import_file_2.save()
tasks.save_raw_data(self.import_file_2.pk)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file_2.id)
tasks.map_data(self.import_file_2.pk)
tasks.match_buildings(self.import_file_2.id)
# for api tests
user_details = {
'username': '******',
'password': '******',
}
self.client.login(**user_details)
def test_match_properties_on_ubid(self):
base_details = {
'ubid': '86HJPCWQ+2VV-1-3-2-3',
'import_file_id': self.import_file.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create set of properties that match each other
self.property_state_factory.get_property_state(**base_details)
base_details['city'] = 'Denver'
self.property_state_factory.get_property_state(**base_details)
# set import_file mapping done so that matching can occur.
self.import_file.mapping_done = True
self.import_file.save()
match_buildings(self.import_file.id)
# 1 Property, 1 PropertyView, 3 PropertyStates (2 imported, 1 merge result)
self.assertEqual(Property.objects.count(), 1)
self.assertEqual(PropertyView.objects.count(), 1)
self.assertEqual(PropertyState.objects.count(), 3)
def test_match_taxlots_normalized_address_used_instead_of_address_line_1(self):
base_details = {
'import_file_id': self.import_file.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create set of taxlots that have the same address_line_1 in slightly different format
base_details['address_line_1'] = '123 Match Street'
self.taxlot_state_factory.get_taxlot_state(**base_details)
base_details['address_line_1'] = '123 match St.'
base_details['city'] = 'Denver'
self.taxlot_state_factory.get_taxlot_state(**base_details)
# set import_file mapping done so that matching can occur.
self.import_file.mapping_done = True
self.import_file.save()
match_buildings(self.import_file.id)
# 1 TaxLot, 1 TaxLotView, 3 TaxLotStates (2 imported, 1 merge result)
self.assertEqual(TaxLot.objects.count(), 1)
self.assertEqual(TaxLotView.objects.count(), 1)
self.assertEqual(TaxLotState.objects.count(), 3)
def start_system_matching(self, request, pk=None):
"""
Starts a background task to attempt automatic matching between buildings
in an ImportFile with other existing buildings within the same org.
---
type:
status:
required: true
type: string
description: either success or error
progress_key:
type: integer
description: ID of background job, for retrieving job progress
parameter_strategy: replace
parameters:
- name: pk
description: Import file ID
required: true
paramType: path
"""
return match_buildings(pk, request.user.pk)
def test_check(self):
# data quality check
d = DataQualityCheck.retrieve(self.org)
d.remove_all_rules()
d.add_rule({
'table_name': 'PropertyState',
'field': 'gross_floor_area',
'data_type': TYPE_NUMBER,
'rule_type': RULE_TYPE_DEFAULT,
'min': 100,
'max': 7000000,
'severity': SEVERITY_ERROR,
'units': 'square feet',
})
d.add_rule({
'table_name': 'PropertyState',
'field': 'recent_sale_date',
'data_type': TYPE_DATE,
'rule_type': RULE_TYPE_DEFAULT,
'min': 18890101,
'max': 20201231,
'severity': SEVERITY_ERROR,
})
# create some status labels for testing
sl_data = {
'name': 'year - old or future',
'super_organization': self.org
}
sl_year, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'year_built',
'data_type': TYPE_YEAR,
'rule_type': RULE_TYPE_DEFAULT,
'min': 1700,
'max': 2019,
'severity': SEVERITY_ERROR,
'status_label': sl_year,
}
d.add_rule(new_rule)
sl_data = {
'name': 'extra data ps float error',
'super_organization': self.org
}
sl_string, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'extra_data_ps_alpha',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'alpha',
'severity': SEVERITY_ERROR,
'units': 'square feet',
'status_label': sl_string,
}
d.add_rule(new_rule)
sl_data = {
'name': 'extra data ps string error',
'super_organization': self.org
}
sl_float, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'extra_data_ps_float',
'data_type': TYPE_NUMBER,
'rule_type': RULE_TYPE_CUSTOM,
'min': 9999,
'max': 10001,
'severity': SEVERITY_ERROR,
'status_label': sl_float,
}
d.add_rule(new_rule)
sl_data = {
'name': 'jurisdiction id does not match',
'super_organization': self.org
}
sl_jurid, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'TaxLotState',
'field': 'jurisdiction_tax_lot_id',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': '1235',
'severity': SEVERITY_ERROR,
'status_label': sl_jurid,
}
d.add_rule(new_rule)
sl_data = {'name': 'No meters present', 'super_organization': self.org}
sl_ok_1, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'Meters Present',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_1,
}
d.add_rule(new_rule)
sl_data = {
'name': 'No 12 Consectutive Months',
'super_organization': self.org
}
sl_ok_2, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': '12 Consectutive Months',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_2,
}
d.add_rule(new_rule)
sl_data = {'name': 'No Monthly Data', 'super_organization': self.org}
sl_ok_3, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'Monthly Data',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_3,
}
d.add_rule(new_rule)
# import data
tasks.save_raw_data(self.import_file.id)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.pk)
tasks.map_data(self.import_file.id)
tasks.match_buildings(self.import_file.id)
qs = PropertyState.objects.filter(
import_file=self.import_file,
source_type=ASSESSED_BS,
).iterator()
d.reset_results()
d.check_data('PropertyState', qs)
result = d.retrieve_result_by_address('4 Myrtle Parkway')
res = [{
"severity": "error",
"value": "27.0",
"field": "extra_data_ps_float",
"table_name": "PropertyState",
"message": "Extra Data Ps Float out of range",
"detailed_message": "Extra Data Ps Float [27.0] < 9999.0",
"formatted_field": "Extra Data Ps Float"
}, {
"severity": "error",
"value": "5.0",
"field": "gross_floor_area",
"table_name": "PropertyState",
"message": "Gross Floor Area out of range",
"detailed_message": "Gross Floor Area [5.0] < 100.0",
"formatted_field": "Gross Floor Area"
}]
self.assertListEqual(result['data_quality_results'], res)
result = d.retrieve_result_by_address('94 Oxford Hill')
res = [{
"severity": "error",
"value": "20000.0",
"field": "extra_data_ps_float",
"table_name": "PropertyState",
"message": "Extra Data Ps Float out of range",
"detailed_message": "Extra Data Ps Float [20000.0] > 10001.0",
"formatted_field": "Extra Data Ps Float"
}, {
"severity": "error",
"value": "1888-01-01 08:00:00",
"field": "recent_sale_date",
"table_name": "PropertyState",
"message": "Recent Sale Date out of range",
"detailed_message":
"Recent Sale Date [1888-01-01 08:00:00] < 1889-01-01 00:00:00",
"formatted_field": "Recent Sale Date"
}]
self.assertListEqual(result['data_quality_results'], res)
result = d.retrieve_result_by_address("3 Portage Alley")
res = [{
'severity': u'error',
'value': 'beta',
'field': u'extra_data_ps_alpha',
'table_name': u'PropertyState',
'message': u'Extra Data Ps Alpha does not match expected value',
'detailed_message':
u'Extra Data Ps Alpha [beta] does not contain "alpha"',
'formatted_field': u'Extra Data Ps Alpha'
}]
self.assertListEqual(result['data_quality_results'], res)
# make sure that the label has been applied
props = PropertyView.objects.filter(
property__labels=sl_year).select_related('state')
addresses = sorted([p.state.address_line_1 for p in props])
expected = sorted([u'84807 Buell Trail', u'1 International Road'])
self.assertListEqual(expected, addresses)
props = PropertyView.objects.filter(
property__labels=sl_float).select_related('state')
addresses = sorted([p.state.address_line_1 for p in props])
expected = sorted([u'4 Myrtle Parkway', u'94 Oxford Hill'])
self.assertListEqual(expected, addresses)
props = PropertyView.objects.filter(
property__labels=sl_string).select_related('state')
addresses = [p.state.address_line_1 for p in props]
expected = [u'3 Portage Alley']
self.assertListEqual(expected, addresses)
# Check tax lots
qs = TaxLotState.objects.filter(
import_file=self.import_file, ).iterator()
d.reset_results()
d.check_data('TaxLotState', qs)
result = d.retrieve_result_by_tax_lot_id("1234")
res = [{
"severity": "error",
"value": "1234",
"field": "jurisdiction_tax_lot_id",
"table_name": "TaxLotState",
"message": "Jurisdiction Tax Lot ID does not match expected value",
"detailed_message":
"Jurisdiction Tax Lot ID [1234] does not contain \"1235\"",
"formatted_field": "Jurisdiction Tax Lot ID"
}]
self.assertListEqual(result['data_quality_results'], res)
# verify labels
taxlots = TaxLotView.objects.filter(
taxlot__labels=sl_jurid).select_related('state')
ids = [t.state.jurisdiction_tax_lot_id for t in taxlots]
expected = '1234'
self.assertEqual(expected, ids[0])
# Check multiple strings
props = PropertyView.objects.filter(
property__labels=sl_ok_1).select_related('state')
addresses = [p.state.address_line_1 for p in props]
def test_taxlots(self):
# Define matching values
matching_jurisdiction_tax_lot_id = '11111'
matching_address_line_1 = '123 Match Street'
matching_ulid = '86HJPCWQ+2VV-1-3-2-3'
matching_custom_id_1 = 'MatchingID12345'
# For first file, create taxlots with no duplicates or matches
base_details_file_1 = {
'import_file_id': self.import_file_1.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# No matching_criteria values
self.taxlot_state_factory.get_taxlot_state(**base_details_file_1)
# Build out taxlots with increasingly more matching_criteria values
base_details_file_1[
'jurisdiction_tax_lot_id'] = matching_jurisdiction_tax_lot_id
self.taxlot_state_factory.get_taxlot_state(**base_details_file_1)
base_details_file_1['address_line_1'] = matching_address_line_1
self.taxlot_state_factory.get_taxlot_state(**base_details_file_1)
base_details_file_1['ulid'] = matching_ulid
self.taxlot_state_factory.get_taxlot_state(**base_details_file_1)
base_details_file_1['custom_id_1'] = matching_custom_id_1
self.taxlot_state_factory.get_taxlot_state(**base_details_file_1)
self.import_file_1.mapping_done = True
self.import_file_1.save()
match_buildings(self.import_file_1.id)
# Verify no duplicates/matched-merges yet
counts = [
TaxLot.objects.count(),
TaxLotState.objects.count(),
TaxLotView.objects.count(),
]
self.assertEqual([5, 5, 5], counts)
"""
For second file, create several taxlots that are one or many of the following:
- duplicates amongst file_1
- duplicates amongst file_2
- matching amongst file_1
- matching amongst file_2
- completely new
"""
base_details_file_2 = {
'import_file_id': self.import_file_2.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create 2 duplicates of the 'No matching_criteria values' taxlots
# (outcome: 2 additional -States, NO new TaxLot/-View)
tls_1 = self.taxlot_state_factory.get_taxlot_state(
**base_details_file_2)
tls_2 = self.taxlot_state_factory.get_taxlot_state(
**base_details_file_2)
# Create 2 completely new taxlots with misaligned combinations of matching values
# (outcome: 2 additional -States, 2 new TaxLot/-View)
base_details_file_2['custom_id_1'] = matching_custom_id_1
tls_3 = self.taxlot_state_factory.get_taxlot_state(
**base_details_file_2)
base_details_file_2['ulid'] = matching_ulid
tls_4 = self.taxlot_state_factory.get_taxlot_state(
**base_details_file_2)
# Create 3 taxlots - with 1 duplicate and 1 match within it's own file that will
# eventually become 1 completely new property
# (outcome: 4 additional -States, 1 new TaxLot/-View)
base_details_file_2['address_line_1'] = matching_address_line_1
base_details_file_2['city'] = 'Denver'
tls_5 = self.taxlot_state_factory.get_taxlot_state(
**base_details_file_2)
tls_6 = self.taxlot_state_factory.get_taxlot_state(
**base_details_file_2)
base_details_file_2['city'] = 'Golden'
tls_7 = self.taxlot_state_factory.get_taxlot_state(
**base_details_file_2)
# Create 3 properties - with 1 duplicate and 1 match within it's own file that will
# eventually match the last property in file_1
# (outcome: 5 additional -States, NO new TaxLot/-View)
base_details_file_2[
'jurisdiction_tax_lot_id'] = matching_jurisdiction_tax_lot_id
base_details_file_2['state'] = 'Colorado'
tls_8 = self.taxlot_state_factory.get_taxlot_state(
**base_details_file_2)
tls_9 = self.taxlot_state_factory.get_taxlot_state(
**base_details_file_2)
base_details_file_2['state'] = 'California'
tls_10 = self.taxlot_state_factory.get_taxlot_state(
**base_details_file_2)
self.import_file_2.mapping_done = True
self.import_file_2.save()
match_buildings(self.import_file_2.id)
self.assertEqual(8, TaxLot.objects.count())
self.assertEqual(8, TaxLotView.objects.count())
self.assertEqual(18, TaxLotState.objects.count())
tls_ids_of_deleted = TaxLotState.objects.filter(
data_state=DATA_STATE_DELETE).values_list('id',
flat=True).order_by('id')
self.assertEqual([tls_1.id, tls_2.id, tls_6.id, tls_9.id],
list(tls_ids_of_deleted))
tls_ids_of_merged_in_file = TaxLotState.objects.filter(
data_state=DATA_STATE_MAPPING,
merge_state=MERGE_STATE_UNKNOWN).values_list(
'id', flat=True).order_by('id')
self.assertEqual([tls_5.id, tls_7.id, tls_8.id, tls_10.id],
list(tls_ids_of_merged_in_file))
tls_ids_of_all_promoted = TaxLotView.objects.values_list('state_id',
flat=True)
self.assertIn(tls_3.id, tls_ids_of_all_promoted)
self.assertIn(tls_4.id, tls_ids_of_all_promoted)
rimport_file_2 = ImportFile.objects.get(pk=self.import_file_2.id)
results = rimport_file_2.matching_results_data
del results['progress_key']
expected = {
'import_file_records':
None, # This is calculated in a separate process
'property_all_unmatched': 0,
'property_duplicates': 0,
'property_duplicates_of_existing': 0,
'property_unmatched': 0,
'tax_lot_all_unmatched': 10,
'tax_lot_duplicates': 3,
'tax_lot_duplicates_of_existing': 1,
'tax_lot_unmatched': 3,
}
self.assertEqual(results, expected)
def test_match_taxlots_if_all_default_fields_match(self):
base_details = {
'address_line_1': '123 Match Street',
'import_file_id': self.import_file_1.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create property in first ImportFile
tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_details)
self.import_file_1.mapping_done = True
self.import_file_1.save()
match_buildings(self.import_file_1.id)
# Create properties from second ImportFile, one matching existing PropertyState
base_details['import_file_id'] = self.import_file_2.id
base_details['city'] = 'Denver'
tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_details)
base_details['jurisdiction_tax_lot_id'] = '11111'
base_details['city'] = 'Philadelphia'
tls_3 = self.taxlot_state_factory.get_taxlot_state(**base_details)
self.import_file_2.mapping_done = True
self.import_file_2.save()
match_buildings(self.import_file_2.id)
# 2 TaxLot, 2 TaxLotViews, 4 TaxLotStates (3 imported, 1 merge result)
self.assertEqual(TaxLot.objects.count(), 2)
self.assertEqual(TaxLotView.objects.count(), 2)
self.assertEqual(TaxLotState.objects.count(), 4)
cities_from_views = []
tls_ids_from_views = []
for tlv in TaxLotView.objects.all():
cities_from_views.append(tlv.state.city)
tls_ids_from_views.append(tlv.state_id)
self.assertIn('Denver', cities_from_views)
self.assertIn('Philadelphia', cities_from_views)
self.assertIn(tls_3.id, tls_ids_from_views)
self.assertNotIn(tls_1.id, tls_ids_from_views)
self.assertNotIn(tls_2.id, tls_ids_from_views)
# Refresh -States and check data_state and merge_state values
rtls_1 = TaxLotState.objects.get(pk=tls_1.id)
self.assertEqual(rtls_1.data_state, DATA_STATE_MATCHING)
self.assertEqual(rtls_1.merge_state, MERGE_STATE_NEW)
rtls_2 = TaxLotState.objects.get(pk=tls_2.id)
self.assertEqual(rtls_2.data_state, DATA_STATE_MATCHING)
self.assertEqual(rtls_2.merge_state, MERGE_STATE_UNKNOWN)
tls_1_plus_2 = TaxLotState.objects.filter(
jurisdiction_tax_lot_id__isnull=True,
city='Denver',
address_line_1='123 Match Street').exclude(
data_state=DATA_STATE_MATCHING,
merge_state=MERGE_STATE_UNKNOWN).get()
self.assertEqual(tls_1_plus_2.data_state, DATA_STATE_MATCHING)
self.assertEqual(tls_1_plus_2.merge_state, MERGE_STATE_MERGED)
rtls_3 = TaxLotState.objects.get(pk=tls_3.id)
self.assertEqual(rtls_3.data_state, DATA_STATE_MATCHING)
self.assertEqual(rtls_3.merge_state, MERGE_STATE_NEW)
def test_merged_indicators_provided_on_filter_endpoint(self):
_import_record, import_file_1 = self.create_import_file(
self.user, self.org, self.cycle)
base_details = {
'address_line_1': '123 Match Street',
'import_file_id': import_file_1.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
self.taxlot_state_factory.get_taxlot_state(**base_details)
# set import_file_1 mapping done so that record is "created for users to view".
import_file_1.mapping_done = True
import_file_1.save()
match_buildings(import_file_1.id)
_import_record_2, import_file_2 = self.create_import_file(
self.user, self.org, self.cycle)
url = reverse(
'api:v2:taxlots-filter'
) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format(
self.cycle.pk, self.org.pk)
response = self.client.post(url)
data = json.loads(response.content)
self.assertFalse(data['results'][0]['merged_indicator'])
# make sure merged_indicator is True when merge occurs
base_details['city'] = 'Denver'
base_details['import_file_id'] = import_file_2.id
self.taxlot_state_factory.get_taxlot_state(**base_details)
# set import_file_2 mapping done so that match merging can occur.
import_file_2.mapping_done = True
import_file_2.save()
match_buildings(import_file_2.id)
url = reverse(
'api:v2:taxlots-filter'
) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format(
self.cycle.pk, self.org.pk)
response = self.client.post(url)
data = json.loads(response.content)
self.assertTrue(data['results'][0]['merged_indicator'])
# Create pairings and check if paired object has indicator as well
property_factory = FakePropertyFactory(organization=self.org)
property_state_factory = FakePropertyStateFactory(
organization=self.org)
property = property_factory.get_property()
property_state = property_state_factory.get_property_state()
property_view = PropertyView.objects.create(property=property,
cycle=self.cycle,
state=property_state)
# attach pairing to one and only taxlot_view
TaxLotProperty(primary=True,
cycle_id=self.cycle.id,
property_view_id=property_view.id,
taxlot_view_id=TaxLotView.objects.get().id).save()
url = reverse(
'api:v2:taxlots-filter'
) + '?cycle_id={}&organization_id={}&page=1&per_page=999999999'.format(
self.cycle.pk, self.org.pk)
response = self.client.post(url)
data = json.loads(response.content)
related = data['results'][0]['related'][0]
self.assertTrue('merged_indicator' in related)
self.assertFalse(related['merged_indicator'])
def test_match_taxlots_if_all_default_fields_match(self):
base_details = {
'address_line_1': '123 Match Street',
'import_file_id': self.import_file.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create first set of taxlots that match each other
tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_details)
base_details['city'] = 'Denver'
tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_details)
# Create second set of taxlots that match each other
base_details['jurisdiction_tax_lot_id'] = '11111'
tls_3 = self.taxlot_state_factory.get_taxlot_state(**base_details)
base_details['city'] = 'Philadelphia'
tls_4 = self.taxlot_state_factory.get_taxlot_state(**base_details)
# Create unmatched taxlot
base_details['jurisdiction_tax_lot_id'] = '000'
tls_5 = self.taxlot_state_factory.get_taxlot_state(**base_details)
# set import_file mapping done so that matching can occur.
self.import_file.mapping_done = True
self.import_file.save()
match_buildings(self.import_file.id)
# 3 TaxLot, 3 TaxLotViews, 7 TaxLotStates (5 imported, 2 merge results)
self.assertEqual(TaxLot.objects.count(), 3)
self.assertEqual(TaxLotView.objects.count(), 3)
self.assertEqual(TaxLotState.objects.count(), 7)
# Refresh -States and check data_state and merge_state values
rtls_1 = TaxLotState.objects.get(pk=tls_1.id)
self.assertEqual(rtls_1.data_state, DATA_STATE_MAPPING)
self.assertEqual(rtls_1.merge_state, MERGE_STATE_UNKNOWN)
rtls_2 = TaxLotState.objects.get(pk=tls_2.id)
self.assertEqual(rtls_2.data_state, DATA_STATE_MAPPING)
self.assertEqual(rtls_2.merge_state, MERGE_STATE_UNKNOWN)
tls_1_plus_2 = TaxLotState.objects.filter(
jurisdiction_tax_lot_id__isnull=True,
city='Denver',
address_line_1='123 Match Street').exclude(
data_state=DATA_STATE_MAPPING,
merge_state=MERGE_STATE_UNKNOWN).get()
self.assertEqual(tls_1_plus_2.data_state, DATA_STATE_MATCHING)
self.assertEqual(tls_1_plus_2.merge_state, MERGE_STATE_MERGED)
rtls_3 = TaxLotState.objects.get(pk=tls_3.id)
self.assertEqual(rtls_3.data_state, DATA_STATE_MAPPING)
self.assertEqual(rtls_3.merge_state, MERGE_STATE_UNKNOWN)
rtls_4 = TaxLotState.objects.get(pk=tls_4.id)
self.assertEqual(rtls_4.data_state, DATA_STATE_MAPPING)
self.assertEqual(rtls_4.merge_state, MERGE_STATE_UNKNOWN)
tls_3_plus_4 = TaxLotState.objects.filter(
jurisdiction_tax_lot_id='11111',
city='Philadelphia',
address_line_1='123 Match Street').exclude(
data_state=DATA_STATE_MAPPING,
merge_state=MERGE_STATE_UNKNOWN).get()
self.assertEqual(tls_3_plus_4.data_state, DATA_STATE_MATCHING)
self.assertEqual(tls_3_plus_4.merge_state, MERGE_STATE_MERGED)
rtls_5 = TaxLotState.objects.get(pk=tls_5.id)
self.assertEqual(rtls_5.data_state, DATA_STATE_MATCHING)
self.assertEqual(rtls_5.merge_state, MERGE_STATE_NEW)
def test_match_properties_if_all_default_fields_match(self):
base_details = {
'address_line_1': '123 Match Street',
'import_file_id': self.import_file.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
# Create first set of properties that match each other
ps_1 = self.property_state_factory.get_property_state(**base_details)
base_details['city'] = 'Denver'
ps_2 = self.property_state_factory.get_property_state(**base_details)
# Create second set of properties that match each other
base_details['pm_property_id'] = '11111'
ps_3 = self.property_state_factory.get_property_state(**base_details)
base_details['city'] = 'Philadelphia'
ps_4 = self.property_state_factory.get_property_state(**base_details)
# Create unmatched property
base_details['pm_property_id'] = '000'
ps_5 = self.property_state_factory.get_property_state(**base_details)
# set import_file mapping done so that matching can occur.
self.import_file.mapping_done = True
self.import_file.save()
match_buildings(self.import_file.id)
# 3 Property, 3 PropertyViews, 7 PropertyStates (5 imported, 2 merge results)
self.assertEqual(Property.objects.count(), 3)
self.assertEqual(PropertyView.objects.count(), 3)
self.assertEqual(PropertyState.objects.count(), 7)
# Refresh -States and check data_state and merge_state values
rps_1 = PropertyState.objects.get(pk=ps_1.id)
self.assertEqual(rps_1.data_state, DATA_STATE_MAPPING)
self.assertEqual(rps_1.merge_state, MERGE_STATE_UNKNOWN)
rps_2 = PropertyState.objects.get(pk=ps_2.id)
self.assertEqual(rps_2.data_state, DATA_STATE_MAPPING)
self.assertEqual(rps_2.merge_state, MERGE_STATE_UNKNOWN)
ps_1_plus_2 = PropertyState.objects.filter(
pm_property_id__isnull=True,
city='Denver',
address_line_1='123 Match Street').exclude(
data_state=DATA_STATE_MAPPING,
merge_state=MERGE_STATE_UNKNOWN).get()
self.assertEqual(ps_1_plus_2.data_state, DATA_STATE_MATCHING)
self.assertEqual(ps_1_plus_2.merge_state, MERGE_STATE_MERGED)
rps_3 = PropertyState.objects.get(pk=ps_3.id)
self.assertEqual(rps_3.data_state, DATA_STATE_MAPPING)
self.assertEqual(rps_3.merge_state, MERGE_STATE_UNKNOWN)
rps_4 = PropertyState.objects.get(pk=ps_4.id)
self.assertEqual(rps_4.data_state, DATA_STATE_MAPPING)
self.assertEqual(rps_4.merge_state, MERGE_STATE_UNKNOWN)
ps_3_plus_4 = PropertyState.objects.filter(
pm_property_id='11111',
city='Philadelphia',
address_line_1='123 Match Street').exclude(
data_state=DATA_STATE_MAPPING,
merge_state=MERGE_STATE_UNKNOWN).get()
self.assertEqual(ps_3_plus_4.data_state, DATA_STATE_MATCHING)
self.assertEqual(ps_3_plus_4.merge_state, MERGE_STATE_MERGED)
rps_5 = PropertyState.objects.get(pk=ps_5.id)
self.assertEqual(rps_5.data_state, DATA_STATE_MATCHING)
self.assertEqual(rps_5.merge_state, MERGE_STATE_NEW)
def test_whole_org_match_merge_link_preview_endpoint_taxlots(self):
# Cycle 1 / ImportFile 1 - Create 1 taxlot
base_taxlot_details = {
'jurisdiction_tax_lot_id': '1st Non-Match Set',
'city': 'City 1',
'district': 'Match Set',
'import_file_id': self.import_file_1.id,
'data_state': DATA_STATE_MAPPING,
'no_default_data': False,
}
tls_1 = self.taxlot_state_factory.get_taxlot_state(**base_taxlot_details)
self.import_file_1.mapping_done = True
self.import_file_1.save()
match_buildings(self.import_file_1.id)
# Cycle 2 / ImportFile 2 - Create 1 unlinked taxlot
base_taxlot_details['jurisdiction_tax_lot_id'] = '2nd Non-Match Set'
base_taxlot_details['district'] = 'Match Set'
base_taxlot_details['import_file_id'] = self.import_file_2.id
tls_2 = self.taxlot_state_factory.get_taxlot_state(**base_taxlot_details)
self.import_file_2.mapping_done = True
self.import_file_2.save()
match_buildings(self.import_file_2.id)
# Check there doesn't exist links
self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id, tls_2.taxlotview_set.first().taxlot_id)
url = reverse('api:v3:organizations-match-merge-link-preview', args=[self.org.id])
post_params = json.dumps({
"inventory_type": "taxlots",
"add": ['district'],
"remove": ['jurisdiction_tax_lot_id']
})
raw_result = self.client.post(url, post_params, content_type='application/json')
# Check there *still* doesn't exist links
self.assertNotEqual(tls_1.taxlotview_set.first().taxlot_id, tls_2.taxlotview_set.first().taxlot_id)
self.assertEqual(200, raw_result.status_code)
raw_content = json.loads(raw_result.content)
identifier = ProgressData.from_key(raw_content['progress_key']).data['unique_id']
result_key = "org_match_merge_link_result__%s" % identifier
raw_summary = get_cache_raw(result_key)
summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles
# Check format of summary
self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys())
# Check that preview shows links would be created
self.assertEqual(summary[str(self.cycle_1.id)][0]['id'], summary[str(self.cycle_2.id)][0]['id'])
# try to get result using results endpoint
get_result_url = reverse('api:v3:organizations-match-merge-link-result', args=[self.org.id]) + '?match_merge_link_id=' + str(identifier)
get_result_raw_response = self.client.get(get_result_url)
raw_summary = json.loads(get_result_raw_response.content)
summary = {str(k): v for k, v in raw_summary.items() if v} # ignore empty cycles
# Check format of summary
self.assertCountEqual([str(self.cycle_1.id), str(self.cycle_2.id)], summary.keys())
# Check that preview shows links would be created
self.assertEqual(summary[str(self.cycle_1.id)][0]['id'], summary[str(self.cycle_2.id)][0]['id'])
def test_demo_v2(self):
tasks._save_raw_data(self.import_file_tax_lot.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_taxlot_mappings, self.org, self.user)
Column.create_mappings(self.fake_portfolio_mappings, self.org, self.user)
tasks.map_data(self.import_file_tax_lot.pk)
# Check to make sure the taxlots were imported
ts = TaxLotState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_tax_lot,
)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
self.assertEqual(len(ps), 0)
self.assertEqual(len(ts), 9)
tasks.match_buildings(self.import_file_tax_lot.id, self.user.id)
# Check a single case of the taxlotstate
self.assertEqual(TaxLotState.objects.filter(address_line_1='050 Willow Ave SE').count(), 1)
self.assertEqual(
TaxLotView.objects.filter(state__address_line_1='050 Willow Ave SE').count(), 1
)
self.assertEqual(TaxLotView.objects.count(), 9)
# Import the property data
tasks._save_raw_data(self.import_file_property.pk, 'fake_cache_key', 1)
tasks.map_data(self.import_file_property.pk)
ts = TaxLotState.objects.filter(
# data_state=DATA_STATE_MAPPING, # Look at all taxlotstates
organization=self.org,
import_file=self.import_file_tax_lot,
)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
self.assertEqual(len(ts), 9)
self.assertEqual(len(ps), 14)
tasks.match_buildings(self.import_file_property.id, self.user.id)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
# there shouldn't be any properties left in the mapping state
self.assertEqual(len(ps), 0)
# psv = PropertyView.objects.filter(state__organization=self.org)
# self.assertEqual(len(psv), 12)
# tlv = TaxLotView.objects.filter(state__organization=self.org)
# self.assertEqual(len(tlv), 9)
self.assertEqual(PropertyView.objects.filter(state__organization=self.org,
state__pm_property_id='2264').count(), 1)
pv = PropertyView.objects.filter(state__organization=self.org,
state__pm_property_id='2264').first()
self.assertEqual(pv.state.property_name, 'University Inn')
self.assertEqual(pv.state.address_line_1, '50 Willow Ave SE')
def test_check_multiple_text_match(self):
d = DataQualityCheck.retrieve(self.org)
d.remove_all_rules()
sl_data = {'name': 'No meters present', 'super_organization': self.org}
sl_ok_1, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'meters_present',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_1,
}
d.add_rule(new_rule)
sl_data = {
'name': 'No 12 Consectutive Months',
'super_organization': self.org
}
sl_ok_2, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': '12 Consectutive Months',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_2,
}
d.add_rule(new_rule)
sl_data = {'name': 'No Monthly Data', 'super_organization': self.org}
sl_ok_3, _ = StatusLabel.objects.get_or_create(**sl_data)
new_rule = {
'table_name': 'PropertyState',
'field': 'Monthly Data',
'data_type': TYPE_STRING,
'rule_type': RULE_TYPE_CUSTOM,
'text_match': 'OK',
'severity': SEVERITY_ERROR,
'status_label': sl_ok_3,
}
d.add_rule(new_rule)
# import data
tasks.save_raw_data(self.import_file.id)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.pk)
tasks.map_data(self.import_file.id)
tasks.match_buildings(self.import_file.id)
qs = PropertyState.objects.filter(
import_file=self.import_file,
source_type=ASSESSED_BS,
).iterator()
d.reset_results()
d.check_data('PropertyState', qs)
# Check multiple strings
props = PropertyView.objects.filter(
property__labels=sl_ok_1).select_related('state')
addresses = sorted([p.state.address_line_1 for p in props])
expected = [
u'1 International Road', u'17246 Esch Drive',
u'2581 Schiller Parkway', u'3 Northport Place',
u'84807 Buell Trail'
]
self.assertListEqual(expected, addresses)
props = PropertyView.objects.filter(
property__labels=sl_ok_2).select_related('state')
addresses = sorted([p.state.address_line_1 for p in props])
expected = [
u'1 International Road', u'2581 Schiller Parkway',
u'49705 Harper Crossing'
]
self.assertListEqual(expected, addresses)
props = PropertyView.objects.filter(
property__labels=sl_ok_3).select_related('state')
addresses = sorted([p.state.address_line_1 for p in props])
expected = [
u'1 International Road', u'17246 Esch Drive', u'84807 Buell Trail',
u'88263 Scoville Park'
]
self.assertListEqual(expected, addresses)
def test_delete_organization_buildings(self):
"""tests the delete buildings for an organization"""
# start with the normal use case
bs1_data = {
'pm_property_id': 123,
# 'tax_lot_id': '435/422',
'property_name': 'Greenfield Complex',
'custom_id_1': 1243,
'address_line_1': '555 NorthWest Databaseer Lane.',
'address_line_2': '',
'city': 'Gotham City',
'postal_code': 8999,
}
# This building will have a lot less data to identify it.
bs2_data = {
'pm_property_id': 1243,
'custom_id_1': 1243,
'address_line_1': '555 Database LN.',
'city': 'Gotham City',
'postal_code': 8999,
}
new_import_file = ImportFile.objects.create(
import_record=self.import_record,
mapping_done=True
)
snapshot = util.make_fake_property(
self.import_file, bs1_data, ASSESSED_BS, is_canon=True
)
snapshot.super_organization = self.fake_org
snapshot.save()
snapshot = util.make_fake_property(
new_import_file,
bs2_data, PORTFOLIO_BS
)
snapshot.super_organization = self.fake_org
snapshot.save()
# TODO: do not import this data to test the deleting
di_tasks.match_buildings(new_import_file.pk, self.fake_user.pk)
# make one more building snapshot in a different org
fake_org_2 = Organization.objects.create()
snapshot = util.make_fake_property(
self.import_file, bs1_data, ASSESSED_BS, is_canon=True
)
snapshot.super_organization = fake_org_2
snapshot.save()
self.assertGreater(BuildingSnapshot.objects.filter(
super_organization=self.fake_org
).count(), 0)
tasks.delete_organization_buildings(self.fake_org.pk,
'fake-progress-key')
self.assertEqual(BuildingSnapshot.objects.filter(
super_organization=self.fake_org
).count(), 0)
self.assertGreater(BuildingSnapshot.objects.filter(
super_organization=fake_org_2
).count(), 0)
# test that the CanonicalBuildings are deleted
self.assertEqual(CanonicalBuilding.objects.filter(
canonical_snapshot__super_organization=self.fake_org
).count(), 0)
# test that other orgs CanonicalBuildings are not deleted
self.assertGreater(CanonicalBuilding.objects.filter(
canonical_snapshot__super_organization=fake_org_2
).count(), 0)
def test_demo_v2(self):
tasks.save_raw_data(self.import_file_tax_lot.pk)
Column.create_mappings(self.fake_taxlot_mappings, self.org, self.user)
Column.create_mappings(self.fake_portfolio_mappings, self.org, self.user)
tasks.map_data(self.import_file_tax_lot.pk)
# Check to make sure the taxlots were imported
ts = TaxLotState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_tax_lot,
)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
self.assertEqual(len(ps), 0)
self.assertEqual(len(ts), 9)
tasks.match_buildings(self.import_file_tax_lot.id)
# Check a single case of the taxlotstate
self.assertEqual(TaxLotState.objects.filter(address_line_1='2655 Welstone Ave NE').count(), 1)
self.assertEqual(
TaxLotView.objects.filter(state__address_line_1='2655 Welstone Ave NE').count(), 1
)
self.assertEqual(TaxLotView.objects.count(), 9)
# Import the property data
tasks.save_raw_data(self.import_file_property.pk)
tasks.map_data(self.import_file_property.pk)
ts = TaxLotState.objects.filter(
# data_state=DATA_STATE_MAPPING, # Look at all taxlotstates
organization=self.org,
import_file=self.import_file_tax_lot,
)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
self.assertEqual(len(ts), 9)
self.assertEqual(len(ps), 14)
tasks.match_buildings(self.import_file_property.id)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
# there should not be any properties left in the mapping state
self.assertEqual(len(ps), 0)
# psv = PropertyView.objects.filter(state__organization=self.org)
# self.assertEqual(len(psv), 12)
# tlv = TaxLotView.objects.filter(state__organization=self.org)
# self.assertEqual(len(tlv), 9)
self.assertEqual(PropertyView.objects.filter(state__organization=self.org,
state__pm_property_id='2264').count(), 1)
pv = PropertyView.objects.filter(state__organization=self.org,
state__pm_property_id='2264').first()
self.assertEqual(pv.state.property_name, 'University Inn')
self.assertEqual(pv.state.address_line_1, '50 Willow Ave SE')
