def test_map_data(self):
"""Save mappings for assessor data based on user specifications."""
fake_import_file = ImportFile.objects.create(
import_record=self.import_record,
raw_save_done=True
)
fake_raw_bs = BuildingSnapshot.objects.create(
import_file=fake_import_file,
source_type=ASSESSED_RAW,
extra_data=self.fake_row
)
util.make_fake_mappings(self.fake_mappings, self.fake_org)
tasks.map_data(fake_import_file.pk)
mapped_bs = list(BuildingSnapshot.objects.filter(
import_file=fake_import_file,
source_type=ASSESSED_BS,
))
self.assertEqual(len(mapped_bs), 1)
test_bs = mapped_bs[0]
self.assertNotEqual(test_bs.pk, fake_raw_bs.pk)
self.assertEqual(test_bs.property_name, self.fake_row['Name'])
self.assertEqual(
test_bs.address_line_1, self.fake_row['Address Line 1']
)
self.assertEqual(
test_bs.year_built,
parser.parse(self.fake_row['Year Built']).year
)
def test_mapping_w_concat(self):
"""When we have a json encoded list as a column mapping, we concat."""
fake_import_file = ImportFile.objects.create(
import_record=self.import_record,
raw_save_done=True
)
self.fake_row['City'] = 'Someplace Nice'
fake_raw_bs = BuildingSnapshot.objects.create(
import_file=fake_import_file,
source_type=ASSESSED_RAW,
extra_data=self.fake_row
)
self.fake_mappings['address_line_1'] = '["Address Line 1", "City"]'
util.make_fake_mappings(self.fake_mappings, self.fake_org)
tasks.map_data(fake_import_file.pk)
mapped_bs = list(BuildingSnapshot.objects.filter(
import_file=fake_import_file,
source_type=ASSESSED_BS,
))[0]
self.assertEqual(
mapped_bs.address_line_1, u'1600 Pennsylvania Ave. Someplace Nice'
)
def test_map_data(self):
"""Save mappings based on user specifications."""
fake_import_file = ImportFile.objects.create(import_record=self.import_record, raw_save_done=True)
fake_raw_bs = BuildingSnapshot.objects.create(
import_file=fake_import_file, extra_data=self.fake_row, source_type=ASSESSED_RAW
)
util.make_fake_mappings(self.fake_mappings, self.fake_org)
tasks.map_data(fake_import_file.pk)
mapped_bs = list(BuildingSnapshot.objects.filter(import_file=fake_import_file, source_type=ASSESSED_BS))
self.assertEqual(len(mapped_bs), 1)
test_bs = mapped_bs[0]
self.assertNotEqual(test_bs.pk, fake_raw_bs.pk)
self.assertEqual(test_bs.property_name, self.fake_row["Name"])
self.assertEqual(test_bs.address_line_1, self.fake_row["Address Line 1"])
self.assertEqual(test_bs.year_built, parser.parse(self.fake_row["Year Built"]).year)
# Make sure that we saved the extra_data column mappings
data_columns = Column.objects.filter(organization=test_bs.super_organization, is_extra_data=True)
# There's only one peice of data that didn't cleanly map
self.assertListEqual(sorted([d.column_name for d in data_columns]), ["Double Tester"])
def test_cleanse(self):
# Import the file and run mapping
# Year Ending,ENERGY STAR Score,Total GHG Emissions (MtCO2e),Weather Normalized Site EUI (kBtu/ft2),
# National Median Site EUI (kBtu/ft2),Source EUI (kBtu/ft2),Weather Normalized Source EUI (kBtu/ft2),
# National Median Source EUI (kBtu/ft2),Parking - Gross Floor Area (ft2),Organization
# Release Date
fake_mappings = {
'pm_property_id': u'Property Id',
'property_name': u'Property Name',
'address_line_1': u'Address 1',
'address_line_2': u'Address 2',
'city': u'City',
'state_province': u'State/Province',
'postal_code': u'Postal Code',
'year_built': u'Year Built',
'gross_floor_area': u'Property Floor Area (Buildings and Parking) (ft2)',
'site_eui': u'Site EUI (kBtu/ft2)',
'generation_date': u'Generation Date'
}
tasks.save_raw_data(self.import_file.id)
util.make_fake_mappings(fake_mappings, self.org)
tasks.map_data(self.import_file.id)
qs = BuildingSnapshot.objects.filter(
import_file=self.import_file,
source_type=PORTFOLIO_BS,
).iterator()
c = Cleansing()
c.cleanse(qs)
data = c.results
self.assertEqual(len(c.results), 2)
result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
if len(result) == 1:
result = result[0]
else:
raise RuntimeError('Non unity results')
res = [{'field': u'pm_property_id', 'message': 'Value is missing', 'severity': 'error'}]
self.assertEqual(res, result['cleansing_results'])
result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
if len(result) == 1:
result = result[0]
else:
raise RuntimeError('Non unity results')
res = [{'field': u'site_eui', 'message': 'Value [0.1] < 10', 'severity': u'warning'}]
self.assertEqual(res, result['cleansing_results'])
def test_mapping_w_concat(self):
"""When we have a json encoded list as a column mapping, we concat."""
fake_import_file = ImportFile.objects.create(
import_record=self.import_record, raw_save_done=True)
self.fake_row['City'] = 'Someplace Nice'
BuildingSnapshot.objects.create(import_file=fake_import_file,
source_type=ASSESSED_RAW,
extra_data=self.fake_row)
self.fake_mappings['address_line_1'] = ['Address Line 1', 'City']
util.make_fake_mappings(self.fake_mappings, self.fake_org)
tasks.map_data(fake_import_file.pk)
mapped_bs = list(
BuildingSnapshot.objects.filter(
import_file=fake_import_file,
source_type=ASSESSED_BS,
))[0]
self.assertEqual(mapped_bs.address_line_1,
u'1600 Pennsylvania Ave. Someplace Nice')
def test_map_data(self):
"""Save mappings based on user specifications."""
fake_import_file = ImportFile.objects.create(
import_record=self.import_record, raw_save_done=True)
fake_raw_bs = BuildingSnapshot.objects.create(
import_file=fake_import_file,
extra_data=self.fake_row,
source_type=ASSESSED_RAW)
util.make_fake_mappings(self.fake_mappings, self.fake_org)
tasks.map_data(fake_import_file.pk)
mapped_bs = list(
BuildingSnapshot.objects.filter(
import_file=fake_import_file,
source_type=ASSESSED_BS,
))
self.assertEqual(len(mapped_bs), 1)
test_bs = mapped_bs[0]
self.assertNotEqual(test_bs.pk, fake_raw_bs.pk)
self.assertEqual(test_bs.property_name, self.fake_row['Name'])
self.assertEqual(test_bs.address_line_1,
self.fake_row['Address Line 1'])
self.assertEqual(test_bs.year_built,
parser.parse(self.fake_row['Year Built']).year)
# Make sure that we saved the extra_data column mappings
data_columns = Column.objects.filter(
organization=test_bs.super_organization, is_extra_data=True)
# There's only one peice of data that didn't cleanly map
self.assertListEqual(sorted([d.column_name for d in data_columns]),
['Double Tester'])
def test_cleanse(self):
# Import the file and run mapping
# This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
# value is the value in the CSV
fake_mappings = {
'city': 'city',
'postal_code': 'Zip',
'gross_floor_area': 'GBA',
'building_count': 'BLDGS',
'year_built': 'AYB_YearBuilt',
'state_province': 'State',
'address_line_1': 'Address',
'owner': 'Owner',
'property_notes': 'Property Type',
'tax_lot_id': 'UBI',
'custom_id_1': 'Custom ID',
'pm_property_id': 'PM Property ID'
}
tasks.save_raw_data(self.import_file.id)
util.make_fake_mappings(fake_mappings, self.org)
tasks.map_data(self.import_file.id)
qs = BuildingSnapshot.objects.filter(
import_file=self.import_file,
source_type=ASSESSED_BS,
).iterator()
c = Cleansing(self.org)
c.cleanse(qs)
data = c.results
self.assertEqual(len(c.results), 2)
result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
if len(result) == 1:
result = result[0]
else:
raise RuntimeError('Non unity results')
self.assertTrue(result['address_line_1'], '95373 E Peach Avenue')
self.assertTrue(result['tax_lot_id'], '10107/c6596')
res = [{
'field': u'pm_property_id',
'formatted_field': u'PM Property ID',
'value': u'',
'message': u'PM Property ID is missing',
'detailed_message': u'PM Property ID is missing',
'severity': u'error'
}]
self.assertEqual(res, result['cleansing_results'])
result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
if len(result) == 1:
result = result[0]
else:
raise RuntimeError('Non unity results')
res = [{
'field': u'year_built',
'formatted_field': u'Year Built',
'value': 0,
'message': u'Year Built out of range',
'detailed_message': u'Year Built [0] < 1700',
'severity': u'error'
}, {
'field': u'gross_floor_area',
'formatted_field': u'Gross Floor Area',
'value': 10000000000.0,
'message': u'Gross Floor Area out of range',
'detailed_message': u'Gross Floor Area [10000000000.0] > 7000000.0',
'severity': u'error'
}, {
'field': u'custom_id_1',
'formatted_field': u'Custom ID 1',
'value': u'',
'message': u'Custom ID 1 is missing',
'detailed_message': u'Custom ID 1 is missing',
'severity': u'error'
}, {
'field': u'pm_property_id',
'formatted_field': u'PM Property ID',
'value': u'',
'message': u'PM Property ID is missing',
'detailed_message': u'PM Property ID is missing',
'severity': u'error'
}]
self.assertItemsEqual(res, result['cleansing_results'])
result = [v for v in c.results.values() if v['address_line_1'] == '1234 Peach Tree Avenue']
self.assertEqual(len(result), 0)
self.assertEqual(result, [])
def test_cleanse(self):
# Import the file and run mapping
# This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
# value is the value in the CSV
fake_mappings = {
'block_number': 'block_number',
'error_type': 'error type',
'building_count': 'building_count',
'conditioned_floor_area': 'conditioned_floor_area',
'energy_score': 'energy_score',
'gross_floor_area': 'gross_floor_area',
'lot_number': 'lot_number',
'occupied_floor_area': 'occupied_floor_area',
'postal_code': 'postal_code',
'site_eui': 'site_eui',
'site_eui_weather_normalized': 'site_eui_weather_normalized',
'source_eui': 'source_eui',
'source_eui_weather_normalized': 'source_eui_weather_normalized',
'address_line_1': 'address_line_1',
'address_line_2': 'address_line_2',
'building_certification': 'building_certification',
'city': 'city',
'custom_id_1': 'custom_id_1',
'district': 'district',
'energy_alerts': 'energy_alerts',
'owner': 'owner',
'owner_address': 'owner_address',
'owner_city_state': 'owner_city_state',
'owner_email': 'owner_email',
'owner_postal_code': 'owner_postal_code',
'owner_telephone': 'owner_telephone',
'pm_property_id': 'pm_property_id',
'property_name': 'property_name',
'property_notes': 'property_notes',
'space_alerts': 'space_alerts',
'state_province': 'state_province',
'tax_lot_id': 'tax_lot_id',
'use_description': 'use_description',
'generation_date': 'generation_date',
'recent_sale_date': 'recent_sale_date',
'release_date': 'release_date',
'year_built': 'year_built',
'year_ending': 'year_ending',
}
tasks.save_raw_data(self.import_file.id)
util.make_fake_mappings(fake_mappings, self.org)
tasks.map_data(self.import_file.id)
qs = BuildingSnapshot.objects.filter(
import_file=self.import_file,
source_type=ASSESSED_BS,
).iterator()
c = Cleansing(self.org)
c.cleanse(qs)
data = c.results
# print data
# This only checks to make sure the 35 errors have occurred.
self.assertEqual(len(c.results), 35)
def test_cleanse(self):
# Import the file and run mapping
# Year Ending,Energy Score,Total GHG Emissions (MtCO2e),Weather Normalized Site EUI (kBtu/ft2),
# National Median Site EUI (kBtu/ft2),Source EUI (kBtu/ft2),Weather Normalized Source EUI (kBtu/ft2),
# National Median Source EUI (kBtu/ft2),Parking - Gross Floor Area (ft2),Organization
# Release Date
fake_mappings = {
'pm_property_id': u'Property Id',
'property_name': u'Property Name',
'address_line_1': u'Address 1',
'address_line_2': u'Address 2',
'city': u'City',
'state_province': u'State/Province',
'postal_code': u'Postal Code',
'year_built': u'Year Built',
'gross_floor_area': u'Property Floor Area (Buildings and Parking) (ft2)',
'site_eui': u'Site EUI (kBtu/ft2)',
'generation_date': u'Generation Date'
}
tasks.save_raw_data(self.import_file.id)
util.make_fake_mappings(fake_mappings, self.org)
tasks.map_data(self.import_file.id)
qs = BuildingSnapshot.objects.filter(
import_file=self.import_file,
source_type=PORTFOLIO_BS,
).iterator()
c = Cleansing(self.org)
c.cleanse(qs)
data = c.results
self.assertEqual(len(c.results), 2)
result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
if len(result) == 1:
result = result[0]
else:
raise RuntimeError('Non unity results')
res = [{
'field': u'pm_property_id',
'formatted_field': u'PM Property ID',
'value': u'',
'message': u'PM Property ID is missing',
'detailed_message': u'PM Property ID is missing',
'severity': u'error'
}]
self.assertEqual(res, result['cleansing_results'])
result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
if len(result) == 1:
result = result[0]
else:
raise RuntimeError('Non unity results')
res = [{
'field': u'site_eui',
'formatted_field': u'Site EUI',
'value': 0.1,
'message': u'Site EUI out of range',
'detailed_message': u'Site EUI [0.1] < 10.0',
'severity': u'warning'
}]
self.assertEqual(res, result['cleansing_results'])
def test_cleanse(self):
# Import the file and run mapping
# This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
# value is the value in the CSV
fake_mappings = {
'city': 'city',
'postal_code': 'Zip',
'gross_floor_area': 'GBA',
'building_count': 'BLDGS',
'year_built': 'AYB_YearBuilt',
'state_province': 'State',
'address_line_1': 'Address',
'owner': 'Owner',
'property_notes': 'Property Type',
'tax_lot_id': 'UBI',
'custom_id_1': 'Custom ID',
'pm_property_id': 'PM Property ID'
}
tasks.save_raw_data(self.import_file.id)
util.make_fake_mappings(fake_mappings, self.org)
tasks.map_data(self.import_file.id)
qs = BuildingSnapshot.objects.filter(
import_file=self.import_file,
source_type=ASSESSED_BS,
).iterator()
c = Cleansing(self.org)
c.cleanse(qs)
self.assertEqual(len(c.results), 2)
result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
if len(result) == 1:
result = result[0]
else:
raise RuntimeError('Non unity results')
self.assertTrue(result['address_line_1'], '95373 E Peach Avenue')
self.assertTrue(result['tax_lot_id'], '10107/c6596')
res = [{
'field': u'pm_property_id',
'formatted_field': u'PM Property ID',
'value': u'',
'message': u'PM Property ID is missing',
'detailed_message': u'PM Property ID is missing',
'severity': u'error'
}]
self.assertEqual(res, result['cleansing_results'])
result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
if len(result) == 1:
result = result[0]
else:
raise RuntimeError('Non unity results')
res = [{
'field': u'year_built',
'formatted_field': u'Year Built',
'value': 0,
'message': u'Year Built out of range',
'detailed_message': u'Year Built [0] < 1700',
'severity': u'error'
}, {
'field': u'gross_floor_area',
'formatted_field': u'Gross Floor Area',
'value': 10000000000.0,
'message': u'Gross Floor Area out of range',
'detailed_message': u'Gross Floor Area [10000000000.0] > 7000000.0',
'severity': u'error'
}, {
'field': u'custom_id_1',
'formatted_field': u'Custom ID 1',
'value': u'',
'message': u'Custom ID 1 is missing',
'detailed_message': u'Custom ID 1 is missing',
'severity': u'error'
}, {
'field': u'pm_property_id',
'formatted_field': u'PM Property ID',
'value': u'',
'message': u'PM Property ID is missing',
'detailed_message': u'PM Property ID is missing',
'severity': u'error'
}]
self.assertItemsEqual(res, result['cleansing_results'])
result = [v for v in c.results.values() if v['address_line_1'] == '1234 Peach Tree Avenue']
self.assertEqual(len(result), 0)
self.assertEqual(result, [])
def test_cleanse(self):
# Import the file and run mapping
# This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
# value is the value in the CSV
fake_mappings = {
'block_number': 'block_number',
'error_type': 'error type',
'building_count': 'building_count',
'conditioned_floor_area': 'conditioned_floor_area',
'energy_score': 'energy_score',
'gross_floor_area': 'gross_floor_area',
'lot_number': 'lot_number',
'occupied_floor_area': 'occupied_floor_area',
'postal_code': 'postal_code',
'site_eui': 'site_eui',
'site_eui_weather_normalized': 'site_eui_weather_normalized',
'source_eui': 'source_eui',
'source_eui_weather_normalized': 'source_eui_weather_normalized',
'address_line_1': 'address_line_1',
'address_line_2': 'address_line_2',
'building_certification': 'building_certification',
'city': 'city',
'custom_id_1': 'custom_id_1',
'district': 'district',
'energy_alerts': 'energy_alerts',
'owner': 'owner',
'owner_address': 'owner_address',
'owner_city_state': 'owner_city_state',
'owner_email': 'owner_email',
'owner_postal_code': 'owner_postal_code',
'owner_telephone': 'owner_telephone',
'pm_property_id': 'pm_property_id',
'property_name': 'property_name',
'property_notes': 'property_notes',
'space_alerts': 'space_alerts',
'state_province': 'state_province',
'tax_lot_id': 'tax_lot_id',
'use_description': 'use_description',
'generation_date': 'generation_date',
'recent_sale_date': 'recent_sale_date',
'release_date': 'release_date',
'year_built': 'year_built',
'year_ending': 'year_ending',
}
tasks.save_raw_data(self.import_file.id)
util.make_fake_mappings(fake_mappings, self.org)
tasks.map_data(self.import_file.id)
qs = BuildingSnapshot.objects.filter(
import_file=self.import_file,
source_type=ASSESSED_BS,
).iterator()
c = Cleansing(self.org)
c.cleanse(qs)
# print data
# This only checks to make sure the 35 errors have occurred.
self.assertEqual(len(c.results), 35)
