def test_import_file(self):
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user)
tasks.map_data(self.import_file.pk)
ps = PropertyState.objects.filter(pm_property_id='2264').first()
ps.promote(self.cycle)
# should only be 11 unmatched_properties because one was promoted.
ps = self.import_file.find_unmatched_property_states()
self.assertEqual(len(ps), 13)
def test_save_raw_data(self):
"""Save information in extra_data, set other attrs."""
with patch.object(ImportFile, 'cache_first_rows', return_value=None):
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
raw_saved = PropertyState.objects.filter(
import_file=self.import_file,
).latest('id')
self.assertDictEqual(raw_saved.extra_data, self.fake_extra_data)
self.assertEqual(raw_saved.organization, self.org)
def test_match_buildings(self):
""" case A (one property <-> one tax lot) """
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user)
tasks.map_data(self.import_file.pk)
# Check to make sure all the properties imported
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file,
)
self.assertEqual(len(ps), 14)
# Check to make sure the taxlots were imported
ts = TaxLotState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file,
)
# self.assertEqual(len(ts), 10) # 10 unique taxlots after duplicates and delimeters
# Check a single case of the taxlotstate
ts = TaxLotState.objects.filter(jurisdiction_tax_lot_id='1552813').first()
self.assertEqual(ts.jurisdiction_tax_lot_id, '1552813')
self.assertEqual(ts.address_line_1, None)
self.assertEqual(ts.extra_data["data_008"], 1)
# Check a single case of the propertystate
ps = PropertyState.objects.filter(pm_property_id='2264')
self.assertEqual(len(ps), 1)
ps = ps.first()
self.assertEqual(ps.pm_property_id, '2264')
self.assertEqual(ps.address_line_1, '50 Willow Ave SE')
self.assertEqual('data_007' in ps.extra_data.keys(), True)
self.assertEqual('data_008' in ps.extra_data.keys(), False)
self.assertEqual(ps.extra_data["data_007"], 'a')
# verify that the lot_number has the tax_lot information. For this case it is one-to-one
self.assertEqual(ps.lot_number, ts.jurisdiction_tax_lot_id)
tasks.match_buildings(self.import_file.id, self.user.id)
self.assertEqual(TaxLot.objects.count(), 10)
qry = PropertyView.objects.filter(state__custom_id_1='7')
self.assertEqual(qry.count(), 1)
state = qry.first().state
self.assertEqual(state.address_line_1, "12 Ninth Street")
self.assertEqual(state.property_name, "Grange Hall")
def test_multiple_id_matches(self):
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user)
tasks.map_data(self.import_file.pk)
# verify that there are no properties listed as canonical
property_states = tasks.list_canonical_property_states(self.org)
self.assertEqual(len(property_states), 0)
# promote two properties
ps = PropertyState.objects.filter(custom_id_1='13')
ps_test = ps.first()
ps_test_2 = ps.last()
for p in ps:
p.promote(self.cycle)
# from seed.utils.generic import pp
# pp(p)
property_states = tasks.list_canonical_property_states(self.org)
self.assertEqual(len(property_states), 2)
# no arguments passed should return no results
matches = tasks.query_property_matches(property_states, None, None)
self.assertEqual(len(matches), 0)
# should return 2 properties
matches = tasks.query_property_matches(property_states, None, '13')
self.assertEqual(len(matches), 2)
self.assertEqual(matches[0], ps_test)
self.assertEqual(matches[1], ps_test_2)
# should return only the second property
matches = tasks.query_property_matches(property_states, '2342', None)
self.assertEqual(len(matches), 1)
self.assertEqual(matches[0], ps_test_2)
# should return both properties, the first one should be the pm match, i.e. the first prop
matches = tasks.query_property_matches(property_states, '481516', '13')
self.assertEqual(len(matches), 2)
self.assertEqual(matches[0], ps_test)
self.assertEqual(matches[1], ps_test_2)
# if passing in the second pm then it will not be the first
matches = tasks.query_property_matches(property_states, '2342', '13')
self.assertEqual(len(matches), 2)
self.assertEqual(matches[1], ps_test_2)
# pass the pm id into the custom id. it should still return the correct buildings.
# not sure that this is the right behavior, but this is what it does, so just testing.
matches = tasks.query_property_matches(property_states, None, '2342')
self.assertEqual(len(matches), 1)
self.assertEqual(matches[0], ps_test_2)
matches = tasks.query_property_matches(property_states, '13', None)
self.assertEqual(len(matches), 2)
self.assertEqual(matches[0], ps_test)
self.assertEqual(matches[1], ps_test_2)
def test_cached_first_row_order(self):
"""Tests to make sure the first row is saved in the correct order.
It should be the order of the headers in the original file."""
with patch.object(ImportFile, 'cache_first_rows', return_value=None):
tasks._save_raw_data(
self.import_file.pk,
'fake_cache_key',
1
)
expected_first_row = u"Property Id|#*#|Property Name|#*#|Year Ending|#*#|Property Floor Area (Buildings and Parking) (ft2)|#*#|Address 1|#*#|Address 2|#*#|City|#*#|State/Province|#*#|Postal Code|#*#|Year Built|#*#|ENERGY STAR Score|#*#|Site EUI (kBtu/ft2)|#*#|Total GHG Emissions (MtCO2e)|#*#|Weather Normalized Site EUI (kBtu/ft2)|#*#|National Median Site EUI (kBtu/ft2)|#*#|Source EUI (kBtu/ft2)|#*#|Weather Normalized Source EUI (kBtu/ft2)|#*#|National Median Source EUI (kBtu/ft2)|#*#|Parking - Gross Floor Area (ft2)|#*#|Organization|#*#|Generation Date|#*#|Release Date" # NOQA
import_file = ImportFile.objects.get(pk=self.import_file.pk)
first_row = import_file.cached_first_row
self.assertEqual(first_row, expected_first_row)
def test_mapping_no_properties(self):
# update the mappings to not include any taxlot tables in the data
for m in self.fake_mappings:
if m["to_table_name"] == 'PropertyState':
m["to_table_name"] = 'TaxLotState'
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user)
tasks.map_data(self.import_file.pk)
# make sure that no taxlot objects were created. the 12 here are the import extra_data.
ps = PropertyState.objects.all()
self.assertEqual(len(ps), 14)
# make sure that the new data was loaded correctly
ts = TaxLotState.objects.filter(address_line_1='50 Willow Ave SE').first()
self.assertEqual(ts.extra_data['site_eui'], 125)
def test_mapping_tax_lots_only(self):
# update the mappings to not include any taxlot tables in the data
new_mappings = copy.deepcopy(self.fake_mappings)
for m in new_mappings:
if m["to_table_name"] == 'PropertyState':
m["to_table_name"] = 'TaxLotState'
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(new_mappings, self.org, self.user, self.import_file.pk)
tasks.map_data(self.import_file.pk)
# make sure that no taxlot objects were created. the 12 here are the import extra_data.
ps = PropertyState.objects.all()
self.assertEqual(len(ps), 14)
# make sure that the new data was loaded correctly
ts = TaxLotState.objects.filter(address_line_1='50 Willow Ave SE').first()
self.assertEqual(ts.extra_data['site_eui'], 125)
def setUp(self):
filename = getattr(self, 'filename',
'example-data-properties-duplicates.xlsx')
import_file_source_type = ASSESSED_RAW
self.fake_mappings = FAKE_MAPPINGS['portfolio']
self.fake_extra_data = FAKE_EXTRA_DATA
self.fake_row = FAKE_ROW
selfvars = self.set_up(import_file_source_type)
self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars
filepath = osp.join(osp.dirname(__file__), 'data', filename)
self.import_file.file = SimpleUploadedFile(name=filename,
content=open(
filepath, 'rb').read())
self.import_file.save()
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.pk)
tasks.map_data(self.import_file.pk)
def test_match_buildings(self):
""" case B (many property <-> one tax lot) """
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.pk)
# Set remap to True because for some reason this file id has been imported before.
tasks.map_data(self.import_file.pk, True)
# Check to make sure all the properties imported
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file,
)
self.assertEqual(len(ps), 14)
# Check to make sure the tax lots were imported
ts = TaxLotState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file,
)
self.assertEqual(len(ts), 18)
# verify that the lot_number has the tax_lot information. For this case it is one-to-many
p_test = PropertyState.objects.filter(
pm_property_id='5233255',
organization=self.org,
data_state=DATA_STATE_MAPPING,
import_file=self.import_file,
).first()
self.assertEqual(p_test.lot_number, "333/66555;333/66125;333/66148")
tasks.match_buildings(self.import_file.id)
# make sure the the property only has one tax lot and vice versa
tlv = TaxLotView.objects.filter(
state__jurisdiction_tax_lot_id='11160509', cycle=self.cycle)
self.assertEqual(len(tlv), 1)
tlv = tlv[0]
properties = tlv.property_states()
self.assertEqual(len(properties), 3)
def test_mapping_no_taxlot(self):
# update the mappings to not include any taxlot tables in the data
# note that save_data reads in from the propertystate table, so that will always
# have entries in the db (for now).
for m in self.fake_mappings:
if m["to_table_name"] == 'TaxLotState':
m["to_table_name"] = 'PropertyState'
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user)
tasks.map_data(self.import_file.pk)
# make sure that no taxlot objects were created
ts = TaxLotState.objects.all()
self.assertEqual(len(ts), 0)
# make sure that the new data was loaded correctly
ps = PropertyState.objects.filter(address_line_1='2700 Welstone Ave NE')[0]
self.assertEqual(ps.site_eui, 1202)
self.assertEqual(ps.extra_data['jurisdiction_tax_lot_id'], '11160509')
def test_mapping(self):
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user)
tasks.map_data(self.import_file.pk)
# There are a total of 18 tax lot ids in the import file
ts = TaxLotState.objects.all()
self.assertEqual(len(ts), 18)
# make sure that the new data was loaded correctly and that the lot_number was set
# appropriately
ps = PropertyState.objects.filter(address_line_1='2700 Welstone Ave NE')[0]
self.assertEqual(ps.site_eui, 1202)
self.assertEqual(ps.lot_number, '11160509')
ps = PropertyState.objects.filter(address_line_1='521 Elm Street')[0]
self.assertEqual(ps.site_eui, 1358)
# The lot_number should also have the normalized code run, then re-delimited
self.assertEqual(ps.lot_number, '33366555;33366125;33366148')
def test_mapping(self):
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user, self.import_file.pk)
tasks.map_data(self.import_file.pk)
# There are a total of 18 tax lot ids in the import file
ts = TaxLotState.objects.all()
self.assertEqual(len(ts), 18)
# make sure that the new data was loaded correctly and that the lot_number was set
# appropriately
ps = PropertyState.objects.filter(address_line_1='2700 Welstone Ave NE')[0]
self.assertEqual(ps.site_eui, 1202)
self.assertEqual(ps.lot_number, '11160509')
ps = PropertyState.objects.filter(address_line_1='521 Elm Street')[0]
self.assertEqual(ps.site_eui, 1358)
# The lot_number should also have the normalized code run, then re-delimited
self.assertEqual(ps.lot_number, '333/66555;333/66125;333/66148')
def test_single_id_matches(self):
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user)
tasks.map_data(self.import_file.pk)
# verify that there are no properties listed as canonical
property_states = tasks.list_canonical_property_states(self.org)
self.assertEqual(len(property_states), 0)
# promote a properties
ps = PropertyState.objects.filter(pm_property_id='2264').first()
ps.promote(self.cycle)
property_states = tasks.list_canonical_property_states(self.org)
self.assertEqual(len(property_states), 1)
matches = tasks.query_property_matches(property_states, None, None)
self.assertEqual(len(matches), 0)
matches = tasks.query_property_matches(property_states, '2264', None)
self.assertEqual(len(matches), 1)
self.assertEqual(matches[0], ps)
def test_mapping_properties_only(self):
# update the mappings to not include any taxlot tables in the data
# note that save_data reads in from the propertystate table, so that will always
# have entries in the db (for now).
new_mappings = copy.deepcopy(self.fake_mappings)
for m in new_mappings:
if m["to_table_name"] == 'TaxLotState':
m["to_table_name"] = 'PropertyState'
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(new_mappings, self.org, self.user, self.import_file.pk)
tasks.map_data(self.import_file.pk)
# make sure that no taxlot objects were created
ts = TaxLotState.objects.all()
self.assertEqual(len(ts), 0)
# make sure that the new data was loaded correctly
ps = PropertyState.objects.filter(address_line_1='2700 Welstone Ave NE')[0]
self.assertEqual(ps.site_eui, 1202)
self.assertEqual(ps.extra_data['jurisdiction_tax_lot_id'], '11160509')
def test_single_id_matches(self):
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user)
tasks.map_data(self.import_file.pk)
# verify that there are no properties listed as canonical
property_states = tasks.list_canonical_property_states(self.org)
self.assertEqual(len(property_states), 0)
# promote a properties
ps = PropertyState.objects.filter(pm_property_id='2264').first()
ps.promote(self.cycle)
property_states = tasks.list_canonical_property_states(self.org)
self.assertEqual(len(property_states), 1)
matches = tasks.query_property_matches(property_states, None, None)
self.assertEqual(len(matches), 0)
matches = tasks.query_property_matches(property_states, '2264', None)
self.assertEqual(len(matches), 1)
self.assertEqual(matches[0], ps)
def setUp(self):
filename = getattr(self, 'filename', 'example-data-properties.xlsx')
self.fake_mappings = copy.copy(FAKE_MAPPINGS['portfolio'])
selfvars = self.set_up(ASSESSED_RAW)
self.user, self.org, self.import_file, self.import_record, self.cycle = selfvars
filepath = osp.join(osp.dirname(__file__), 'data', filename)
self.import_file.file = SimpleUploadedFile(name=filename,
content=open(
filepath, 'rb').read())
self.import_file.save()
tasks._save_raw_data(self.import_file.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file.id)
tasks.map_data(self.import_file.pk)
tasks.match_buildings(self.import_file.id)
# import second file that is currently the same, but should be slightly different
filename_2 = getattr(self, 'filename',
'example-data-properties-small-changes.xlsx')
_, self.import_file_2 = self.create_import_file(
self.user, self.org, self.cycle)
filepath = osp.join(osp.dirname(__file__), 'data', filename_2)
self.import_file_2.file = SimpleUploadedFile(name=filename_2,
content=open(
filepath,
'rb').read())
self.import_file_2.save()
tasks._save_raw_data(self.import_file_2.pk, 'fake_cache_key_2', 1)
Column.create_mappings(self.fake_mappings, self.org, self.user,
self.import_file_2.id)
tasks.map_data(self.import_file_2.pk)
tasks.match_buildings(self.import_file_2.id)
# for api tests
user_details = {
'username': '******',
'password': '******',
}
self.client.login(**user_details)
def test_demo_v2(self):
tasks._save_raw_data(self.import_file_tax_lot.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_taxlot_mappings, self.org, self.user)
Column.create_mappings(self.fake_portfolio_mappings, self.org, self.user)
tasks.map_data(self.import_file_tax_lot.pk)
# Check to make sure the taxlots were imported
ts = TaxLotState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_tax_lot,
)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
self.assertEqual(len(ps), 0)
self.assertEqual(len(ts), 9)
tasks.match_buildings(self.import_file_tax_lot.id)
# Check a single case of the taxlotstate
self.assertEqual(TaxLotState.objects.filter(address_line_1='050 Willow Ave SE').count(), 1)
self.assertEqual(
TaxLotView.objects.filter(state__address_line_1='050 Willow Ave SE').count(), 1
)
self.assertEqual(TaxLotView.objects.count(), 9)
# Import the property data
tasks._save_raw_data(self.import_file_property.pk, 'fake_cache_key', 1)
tasks.map_data(self.import_file_property.pk)
ts = TaxLotState.objects.filter(
# data_state=DATA_STATE_MAPPING, # Look at all taxlotstates
organization=self.org,
import_file=self.import_file_tax_lot,
)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
self.assertEqual(len(ts), 9)
self.assertEqual(len(ps), 14)
tasks.match_buildings(self.import_file_property.id)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
# there should not be any properties left in the mapping state
self.assertEqual(len(ps), 0)
# psv = PropertyView.objects.filter(state__organization=self.org)
# self.assertEqual(len(psv), 12)
# tlv = TaxLotView.objects.filter(state__organization=self.org)
# self.assertEqual(len(tlv), 9)
self.assertEqual(PropertyView.objects.filter(state__organization=self.org,
state__pm_property_id='2264').count(), 1)
pv = PropertyView.objects.filter(state__organization=self.org,
state__pm_property_id='2264').first()
self.assertEqual(pv.state.property_name, 'University Inn')
self.assertEqual(pv.state.address_line_1, '50 Willow Ave SE')
def test_demo_v2(self):
tasks._save_raw_data(self.import_file_tax_lot.pk, 'fake_cache_key', 1)
Column.create_mappings(self.fake_taxlot_mappings, self.org, self.user)
Column.create_mappings(self.fake_portfolio_mappings, self.org, self.user)
tasks.map_data(self.import_file_tax_lot.pk)
# Check to make sure the taxlots were imported
ts = TaxLotState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_tax_lot,
)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
self.assertEqual(len(ps), 0)
self.assertEqual(len(ts), 9)
tasks.match_buildings(self.import_file_tax_lot.id, self.user.id)
# Check a single case of the taxlotstate
self.assertEqual(TaxLotState.objects.filter(address_line_1='050 Willow Ave SE').count(), 1)
self.assertEqual(
TaxLotView.objects.filter(state__address_line_1='050 Willow Ave SE').count(), 1
)
self.assertEqual(TaxLotView.objects.count(), 9)
# Import the property data
tasks._save_raw_data(self.import_file_property.pk, 'fake_cache_key', 1)
tasks.map_data(self.import_file_property.pk)
ts = TaxLotState.objects.filter(
# data_state=DATA_STATE_MAPPING, # Look at all taxlotstates
organization=self.org,
import_file=self.import_file_tax_lot,
)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
self.assertEqual(len(ts), 9)
self.assertEqual(len(ps), 14)
tasks.match_buildings(self.import_file_property.id, self.user.id)
ps = PropertyState.objects.filter(
data_state=DATA_STATE_MAPPING,
organization=self.org,
import_file=self.import_file_property,
)
# there shouldn't be any properties left in the mapping state
self.assertEqual(len(ps), 0)
# psv = PropertyView.objects.filter(state__organization=self.org)
# self.assertEqual(len(psv), 12)
# tlv = TaxLotView.objects.filter(state__organization=self.org)
# self.assertEqual(len(tlv), 9)
self.assertEqual(PropertyView.objects.filter(state__organization=self.org,
state__pm_property_id='2264').count(), 1)
pv = PropertyView.objects.filter(state__organization=self.org,
state__pm_property_id='2264').first()
self.assertEqual(pv.state.property_name, 'University Inn')
self.assertEqual(pv.state.address_line_1, '50 Willow Ave SE')
