Exemple #1
0
    def test_map_data(self):
        """Save mappings for assessor data based on user specifications."""
        fake_import_file = ImportFile.objects.create(
            import_record=self.import_record,
            raw_save_done=True
        )
        fake_raw_bs = BuildingSnapshot.objects.create(
            import_file=fake_import_file,
            source_type=ASSESSED_RAW,
            extra_data=self.fake_row
        )

        util.make_fake_mappings(self.fake_mappings, self.fake_org)

        tasks.map_data(fake_import_file.pk)

        mapped_bs = list(BuildingSnapshot.objects.filter(
            import_file=fake_import_file,
            source_type=ASSESSED_BS,
        ))

        self.assertEqual(len(mapped_bs), 1)

        test_bs = mapped_bs[0]

        self.assertNotEqual(test_bs.pk, fake_raw_bs.pk)
        self.assertEqual(test_bs.property_name, self.fake_row['Name'])
        self.assertEqual(
            test_bs.address_line_1, self.fake_row['Address Line 1']
        )
        self.assertEqual(
            test_bs.year_built,
            parser.parse(self.fake_row['Year Built']).year
        )
Exemple #2
0
    def test_mapping_w_concat(self):
        """When we have a json encoded list as a column mapping, we concat."""
        fake_import_file = ImportFile.objects.create(
            import_record=self.import_record,
            raw_save_done=True
        )
        self.fake_row['City'] = 'Someplace Nice'
        fake_raw_bs = BuildingSnapshot.objects.create(
            import_file=fake_import_file,
            source_type=ASSESSED_RAW,
            extra_data=self.fake_row
        )

        self.fake_mappings['address_line_1'] = '["Address Line 1", "City"]'
        util.make_fake_mappings(self.fake_mappings, self.fake_org)

        tasks.map_data(fake_import_file.pk)

        mapped_bs = list(BuildingSnapshot.objects.filter(
            import_file=fake_import_file,
            source_type=ASSESSED_BS,
        ))[0]

        self.assertEqual(
            mapped_bs.address_line_1, u'1600 Pennsylvania Ave. Someplace Nice'
        )
Exemple #3
0
    def test_map_data(self):
        """Save mappings based on user specifications."""
        fake_import_file = ImportFile.objects.create(import_record=self.import_record, raw_save_done=True)
        fake_raw_bs = BuildingSnapshot.objects.create(
            import_file=fake_import_file, extra_data=self.fake_row, source_type=ASSESSED_RAW
        )

        util.make_fake_mappings(self.fake_mappings, self.fake_org)

        tasks.map_data(fake_import_file.pk)

        mapped_bs = list(BuildingSnapshot.objects.filter(import_file=fake_import_file, source_type=ASSESSED_BS))

        self.assertEqual(len(mapped_bs), 1)

        test_bs = mapped_bs[0]

        self.assertNotEqual(test_bs.pk, fake_raw_bs.pk)
        self.assertEqual(test_bs.property_name, self.fake_row["Name"])
        self.assertEqual(test_bs.address_line_1, self.fake_row["Address Line 1"])
        self.assertEqual(test_bs.year_built, parser.parse(self.fake_row["Year Built"]).year)

        # Make sure that we saved the extra_data column mappings
        data_columns = Column.objects.filter(organization=test_bs.super_organization, is_extra_data=True)

        # There's only one peice of data that didn't cleanly map
        self.assertListEqual(sorted([d.column_name for d in data_columns]), ["Double Tester"])
Exemple #4
0
    def test_cleanse(self):
        # Import the file and run mapping

        # Year Ending,ENERGY STAR Score,Total GHG Emissions (MtCO2e),Weather Normalized Site EUI (kBtu/ft2),
        # National Median Site EUI (kBtu/ft2),Source EUI (kBtu/ft2),Weather Normalized Source EUI (kBtu/ft2),
        # National Median Source EUI (kBtu/ft2),Parking - Gross Floor Area (ft2),Organization
        # Release Date
        fake_mappings = {
            'pm_property_id': u'Property Id',
            'property_name': u'Property Name',
            'address_line_1': u'Address 1',
            'address_line_2': u'Address 2',
            'city': u'City',
            'state_province': u'State/Province',
            'postal_code': u'Postal Code',
            'year_built': u'Year Built',
            'gross_floor_area': u'Property Floor Area (Buildings and Parking) (ft2)',
            'site_eui': u'Site EUI (kBtu/ft2)',
            'generation_date': u'Generation Date'
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=PORTFOLIO_BS,
        ).iterator()

        c = Cleansing()
        c.cleanse(qs)

        data = c.results

        self.assertEqual(len(c.results), 2)

        result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{'field': u'pm_property_id', 'message': 'Value is missing', 'severity': 'error'}]
        self.assertEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{'field': u'site_eui', 'message': 'Value [0.1] < 10', 'severity': u'warning'}]
        self.assertEqual(res, result['cleansing_results'])
Exemple #5
0
    def test_mapping_w_concat(self):
        """When we have a json encoded list as a column mapping, we concat."""
        fake_import_file = ImportFile.objects.create(
            import_record=self.import_record, raw_save_done=True)
        self.fake_row['City'] = 'Someplace Nice'
        BuildingSnapshot.objects.create(import_file=fake_import_file,
                                        source_type=ASSESSED_RAW,
                                        extra_data=self.fake_row)

        self.fake_mappings['address_line_1'] = ['Address Line 1', 'City']
        util.make_fake_mappings(self.fake_mappings, self.fake_org)

        tasks.map_data(fake_import_file.pk)

        mapped_bs = list(
            BuildingSnapshot.objects.filter(
                import_file=fake_import_file,
                source_type=ASSESSED_BS,
            ))[0]

        self.assertEqual(mapped_bs.address_line_1,
                         u'1600 Pennsylvania Ave. Someplace Nice')
Exemple #6
0
    def test_map_data(self):
        """Save mappings based on user specifications."""
        fake_import_file = ImportFile.objects.create(
            import_record=self.import_record, raw_save_done=True)
        fake_raw_bs = BuildingSnapshot.objects.create(
            import_file=fake_import_file,
            extra_data=self.fake_row,
            source_type=ASSESSED_RAW)

        util.make_fake_mappings(self.fake_mappings, self.fake_org)

        tasks.map_data(fake_import_file.pk)

        mapped_bs = list(
            BuildingSnapshot.objects.filter(
                import_file=fake_import_file,
                source_type=ASSESSED_BS,
            ))

        self.assertEqual(len(mapped_bs), 1)

        test_bs = mapped_bs[0]

        self.assertNotEqual(test_bs.pk, fake_raw_bs.pk)
        self.assertEqual(test_bs.property_name, self.fake_row['Name'])
        self.assertEqual(test_bs.address_line_1,
                         self.fake_row['Address Line 1'])
        self.assertEqual(test_bs.year_built,
                         parser.parse(self.fake_row['Year Built']).year)

        # Make sure that we saved the extra_data column mappings
        data_columns = Column.objects.filter(
            organization=test_bs.super_organization, is_extra_data=True)

        # There's only one peice of data that didn't cleanly map
        self.assertListEqual(sorted([d.column_name for d in data_columns]),
                             ['Double Tester'])
Exemple #7
0
    def test_cleanse(self):
        # Import the file and run mapping

        # This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
        # value is the value in the CSV
        fake_mappings = {
            'city': 'city',
            'postal_code': 'Zip',
            'gross_floor_area': 'GBA',
            'building_count': 'BLDGS',
            'year_built': 'AYB_YearBuilt',
            'state_province': 'State',
            'address_line_1': 'Address',
            'owner': 'Owner',
            'property_notes': 'Property Type',
            'tax_lot_id': 'UBI',
            'custom_id_1': 'Custom ID',
            'pm_property_id': 'PM Property ID'
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=ASSESSED_BS,
        ).iterator()

        c = Cleansing(self.org)
        c.cleanse(qs)

        data = c.results

        self.assertEqual(len(c.results), 2)

        result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        self.assertTrue(result['address_line_1'], '95373 E Peach Avenue')
        self.assertTrue(result['tax_lot_id'], '10107/c6596')
        res = [{
            'field': u'pm_property_id',
            'formatted_field': u'PM Property ID',
            'value': u'',
            'message': u'PM Property ID is missing',
            'detailed_message': u'PM Property ID is missing',
            'severity': u'error'
        }]
        self.assertEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{
            'field': u'year_built',
            'formatted_field': u'Year Built',
            'value': 0,
            'message': u'Year Built out of range',
            'detailed_message': u'Year Built [0] < 1700',
            'severity': u'error'
        }, {
            'field': u'gross_floor_area',
            'formatted_field': u'Gross Floor Area',
            'value': 10000000000.0,
            'message': u'Gross Floor Area out of range',
            'detailed_message': u'Gross Floor Area [10000000000.0] > 7000000.0',
            'severity': u'error'
        }, {
            'field': u'custom_id_1',
            'formatted_field': u'Custom ID 1',
            'value': u'',
            'message': u'Custom ID 1 is missing',
            'detailed_message': u'Custom ID 1 is missing',
            'severity': u'error'
        }, {
            'field': u'pm_property_id',
            'formatted_field': u'PM Property ID',
            'value': u'',
            'message': u'PM Property ID is missing',
            'detailed_message': u'PM Property ID is missing',
            'severity': u'error'
        }]
        self.assertItemsEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '1234 Peach Tree Avenue']
        self.assertEqual(len(result), 0)
        self.assertEqual(result, [])
Exemple #8
0
    def test_cleanse(self):
        # Import the file and run mapping

        # This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
        # value is the value in the CSV
        fake_mappings = {
            'block_number': 'block_number',
            'error_type': 'error type',
            'building_count': 'building_count',
            'conditioned_floor_area': 'conditioned_floor_area',
            'energy_score': 'energy_score',
            'gross_floor_area': 'gross_floor_area',
            'lot_number': 'lot_number',
            'occupied_floor_area': 'occupied_floor_area',
            'postal_code': 'postal_code',
            'site_eui': 'site_eui',
            'site_eui_weather_normalized': 'site_eui_weather_normalized',
            'source_eui': 'source_eui',
            'source_eui_weather_normalized': 'source_eui_weather_normalized',
            'address_line_1': 'address_line_1',
            'address_line_2': 'address_line_2',
            'building_certification': 'building_certification',
            'city': 'city',
            'custom_id_1': 'custom_id_1',
            'district': 'district',
            'energy_alerts': 'energy_alerts',
            'owner': 'owner',
            'owner_address': 'owner_address',
            'owner_city_state': 'owner_city_state',
            'owner_email': 'owner_email',
            'owner_postal_code': 'owner_postal_code',
            'owner_telephone': 'owner_telephone',
            'pm_property_id': 'pm_property_id',
            'property_name': 'property_name',
            'property_notes': 'property_notes',
            'space_alerts': 'space_alerts',
            'state_province': 'state_province',
            'tax_lot_id': 'tax_lot_id',
            'use_description': 'use_description',
            'generation_date': 'generation_date',
            'recent_sale_date': 'recent_sale_date',
            'release_date': 'release_date',
            'year_built': 'year_built',
            'year_ending': 'year_ending',
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=ASSESSED_BS,
        ).iterator()

        c = Cleansing(self.org)
        c.cleanse(qs)

        data = c.results

        # print data
        # This only checks to make sure the 35 errors have occurred.
        self.assertEqual(len(c.results), 35)
Exemple #9
0
    def test_cleanse(self):
        # Import the file and run mapping

        # Year Ending,Energy Score,Total GHG Emissions (MtCO2e),Weather Normalized Site EUI (kBtu/ft2),
        # National Median Site EUI (kBtu/ft2),Source EUI (kBtu/ft2),Weather Normalized Source EUI (kBtu/ft2),
        # National Median Source EUI (kBtu/ft2),Parking - Gross Floor Area (ft2),Organization
        # Release Date
        fake_mappings = {
            'pm_property_id': u'Property Id',
            'property_name': u'Property Name',
            'address_line_1': u'Address 1',
            'address_line_2': u'Address 2',
            'city': u'City',
            'state_province': u'State/Province',
            'postal_code': u'Postal Code',
            'year_built': u'Year Built',
            'gross_floor_area': u'Property Floor Area (Buildings and Parking) (ft2)',
            'site_eui': u'Site EUI (kBtu/ft2)',
            'generation_date': u'Generation Date'
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=PORTFOLIO_BS,
        ).iterator()

        c = Cleansing(self.org)
        c.cleanse(qs)

        data = c.results

        self.assertEqual(len(c.results), 2)

        result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{
            'field': u'pm_property_id',
            'formatted_field': u'PM Property ID',
            'value': u'',
            'message': u'PM Property ID is missing',
            'detailed_message': u'PM Property ID is missing',
            'severity': u'error'
        }]
        self.assertEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{
            'field': u'site_eui',
            'formatted_field': u'Site EUI',
            'value': 0.1,
            'message': u'Site EUI out of range',
            'detailed_message': u'Site EUI [0.1] < 10.0',
            'severity': u'warning'
        }]
        self.assertEqual(res, result['cleansing_results'])
Exemple #10
0
    def test_cleanse(self):
        # Import the file and run mapping

        # This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
        # value is the value in the CSV
        fake_mappings = {
            'city': 'city',
            'postal_code': 'Zip',
            'gross_floor_area': 'GBA',
            'building_count': 'BLDGS',
            'year_built': 'AYB_YearBuilt',
            'state_province': 'State',
            'address_line_1': 'Address',
            'owner': 'Owner',
            'property_notes': 'Property Type',
            'tax_lot_id': 'UBI',
            'custom_id_1': 'Custom ID',
            'pm_property_id': 'PM Property ID'
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=ASSESSED_BS,
        ).iterator()

        c = Cleansing(self.org)
        c.cleanse(qs)

        self.assertEqual(len(c.results), 2)

        result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        self.assertTrue(result['address_line_1'], '95373 E Peach Avenue')
        self.assertTrue(result['tax_lot_id'], '10107/c6596')
        res = [{
            'field': u'pm_property_id',
            'formatted_field': u'PM Property ID',
            'value': u'',
            'message': u'PM Property ID is missing',
            'detailed_message': u'PM Property ID is missing',
            'severity': u'error'
        }]
        self.assertEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{
            'field': u'year_built',
            'formatted_field': u'Year Built',
            'value': 0,
            'message': u'Year Built out of range',
            'detailed_message': u'Year Built [0] < 1700',
            'severity': u'error'
        }, {
            'field': u'gross_floor_area',
            'formatted_field': u'Gross Floor Area',
            'value': 10000000000.0,
            'message': u'Gross Floor Area out of range',
            'detailed_message': u'Gross Floor Area [10000000000.0] > 7000000.0',
            'severity': u'error'
        }, {
            'field': u'custom_id_1',
            'formatted_field': u'Custom ID 1',
            'value': u'',
            'message': u'Custom ID 1 is missing',
            'detailed_message': u'Custom ID 1 is missing',
            'severity': u'error'
        }, {
            'field': u'pm_property_id',
            'formatted_field': u'PM Property ID',
            'value': u'',
            'message': u'PM Property ID is missing',
            'detailed_message': u'PM Property ID is missing',
            'severity': u'error'
        }]
        self.assertItemsEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '1234 Peach Tree Avenue']
        self.assertEqual(len(result), 0)
        self.assertEqual(result, [])
Exemple #11
0
    def test_cleanse(self):
        # Import the file and run mapping

        # This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
        # value is the value in the CSV
        fake_mappings = {
            'block_number': 'block_number',
            'error_type': 'error type',
            'building_count': 'building_count',
            'conditioned_floor_area': 'conditioned_floor_area',
            'energy_score': 'energy_score',
            'gross_floor_area': 'gross_floor_area',
            'lot_number': 'lot_number',
            'occupied_floor_area': 'occupied_floor_area',
            'postal_code': 'postal_code',
            'site_eui': 'site_eui',
            'site_eui_weather_normalized': 'site_eui_weather_normalized',
            'source_eui': 'source_eui',
            'source_eui_weather_normalized': 'source_eui_weather_normalized',
            'address_line_1': 'address_line_1',
            'address_line_2': 'address_line_2',
            'building_certification': 'building_certification',
            'city': 'city',
            'custom_id_1': 'custom_id_1',
            'district': 'district',
            'energy_alerts': 'energy_alerts',
            'owner': 'owner',
            'owner_address': 'owner_address',
            'owner_city_state': 'owner_city_state',
            'owner_email': 'owner_email',
            'owner_postal_code': 'owner_postal_code',
            'owner_telephone': 'owner_telephone',
            'pm_property_id': 'pm_property_id',
            'property_name': 'property_name',
            'property_notes': 'property_notes',
            'space_alerts': 'space_alerts',
            'state_province': 'state_province',
            'tax_lot_id': 'tax_lot_id',
            'use_description': 'use_description',
            'generation_date': 'generation_date',
            'recent_sale_date': 'recent_sale_date',
            'release_date': 'release_date',
            'year_built': 'year_built',
            'year_ending': 'year_ending',
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=ASSESSED_BS,
        ).iterator()

        c = Cleansing(self.org)
        c.cleanse(qs)

        # print data
        # This only checks to make sure the 35 errors have occurred.
        self.assertEqual(len(c.results), 35)