Пример #1
0
    def test_save_raw_data_batch_iterator(self):
        """Ensure split_csv completes"""
        tasks.save_raw_data(self.import_file.pk)

        self.assertEqual(
            BuildingSnapshot.objects.filter(
                import_file=self.import_file).count(), 512)
Пример #2
0
    def test_save_raw_data_batch_iterator(self):
        """Ensure split_csv completes"""
        tasks.save_raw_data(self.import_file.pk)

        self.assertEqual(BuildingSnapshot.objects.filter(
            import_file=self.import_file
        ).count(), 512)
Пример #3
0
    def test_cleanse(self):
        # Import the file and run mapping

        # Year Ending,ENERGY STAR Score,Total GHG Emissions (MtCO2e),Weather Normalized Site EUI (kBtu/ft2),
        # National Median Site EUI (kBtu/ft2),Source EUI (kBtu/ft2),Weather Normalized Source EUI (kBtu/ft2),
        # National Median Source EUI (kBtu/ft2),Parking - Gross Floor Area (ft2),Organization
        # Release Date
        fake_mappings = {
            'pm_property_id': u'Property Id',
            'property_name': u'Property Name',
            'address_line_1': u'Address 1',
            'address_line_2': u'Address 2',
            'city': u'City',
            'state_province': u'State/Province',
            'postal_code': u'Postal Code',
            'year_built': u'Year Built',
            'gross_floor_area': u'Property Floor Area (Buildings and Parking) (ft2)',
            'site_eui': u'Site EUI (kBtu/ft2)',
            'generation_date': u'Generation Date'
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=PORTFOLIO_BS,
        ).iterator()

        c = Cleansing()
        c.cleanse(qs)

        data = c.results

        self.assertEqual(len(c.results), 2)

        result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{'field': u'pm_property_id', 'message': 'Value is missing', 'severity': 'error'}]
        self.assertEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{'field': u'site_eui', 'message': 'Value [0.1] < 10', 'severity': u'warning'}]
        self.assertEqual(res, result['cleansing_results'])
Пример #4
0
    def test_cleanse(self):
        # Import the file and run mapping

        # This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
        # value is the value in the CSV
        fake_mappings = {
            'city': 'city',
            'postal_code': 'Zip',
            'gross_floor_area': 'GBA',
            'building_count': 'BLDGS',
            'year_built': 'AYB_YearBuilt',
            'state_province': 'State',
            'address_line_1': 'Address',
            'owner': 'Owner',
            'property_notes': 'Property Type',
            'tax_lot_id': 'UBI',
            'custom_id_1': 'Custom ID',
            'pm_property_id': 'PM Property ID'
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=ASSESSED_BS,
        ).iterator()

        c = Cleansing(self.org)
        c.cleanse(qs)

        data = c.results

        self.assertEqual(len(c.results), 2)

        result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        self.assertTrue(result['address_line_1'], '95373 E Peach Avenue')
        self.assertTrue(result['tax_lot_id'], '10107/c6596')
        res = [{
            'field': u'pm_property_id',
            'formatted_field': u'PM Property ID',
            'value': u'',
            'message': u'PM Property ID is missing',
            'detailed_message': u'PM Property ID is missing',
            'severity': u'error'
        }]
        self.assertEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{
            'field': u'year_built',
            'formatted_field': u'Year Built',
            'value': 0,
            'message': u'Year Built out of range',
            'detailed_message': u'Year Built [0] < 1700',
            'severity': u'error'
        }, {
            'field': u'gross_floor_area',
            'formatted_field': u'Gross Floor Area',
            'value': 10000000000.0,
            'message': u'Gross Floor Area out of range',
            'detailed_message': u'Gross Floor Area [10000000000.0] > 7000000.0',
            'severity': u'error'
        }, {
            'field': u'custom_id_1',
            'formatted_field': u'Custom ID 1',
            'value': u'',
            'message': u'Custom ID 1 is missing',
            'detailed_message': u'Custom ID 1 is missing',
            'severity': u'error'
        }, {
            'field': u'pm_property_id',
            'formatted_field': u'PM Property ID',
            'value': u'',
            'message': u'PM Property ID is missing',
            'detailed_message': u'PM Property ID is missing',
            'severity': u'error'
        }]
        self.assertItemsEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '1234 Peach Tree Avenue']
        self.assertEqual(len(result), 0)
        self.assertEqual(result, [])
Пример #5
0
    def test_cleanse(self):
        # Import the file and run mapping

        # This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
        # value is the value in the CSV
        fake_mappings = {
            'block_number': 'block_number',
            'error_type': 'error type',
            'building_count': 'building_count',
            'conditioned_floor_area': 'conditioned_floor_area',
            'energy_score': 'energy_score',
            'gross_floor_area': 'gross_floor_area',
            'lot_number': 'lot_number',
            'occupied_floor_area': 'occupied_floor_area',
            'postal_code': 'postal_code',
            'site_eui': 'site_eui',
            'site_eui_weather_normalized': 'site_eui_weather_normalized',
            'source_eui': 'source_eui',
            'source_eui_weather_normalized': 'source_eui_weather_normalized',
            'address_line_1': 'address_line_1',
            'address_line_2': 'address_line_2',
            'building_certification': 'building_certification',
            'city': 'city',
            'custom_id_1': 'custom_id_1',
            'district': 'district',
            'energy_alerts': 'energy_alerts',
            'owner': 'owner',
            'owner_address': 'owner_address',
            'owner_city_state': 'owner_city_state',
            'owner_email': 'owner_email',
            'owner_postal_code': 'owner_postal_code',
            'owner_telephone': 'owner_telephone',
            'pm_property_id': 'pm_property_id',
            'property_name': 'property_name',
            'property_notes': 'property_notes',
            'space_alerts': 'space_alerts',
            'state_province': 'state_province',
            'tax_lot_id': 'tax_lot_id',
            'use_description': 'use_description',
            'generation_date': 'generation_date',
            'recent_sale_date': 'recent_sale_date',
            'release_date': 'release_date',
            'year_built': 'year_built',
            'year_ending': 'year_ending',
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=ASSESSED_BS,
        ).iterator()

        c = Cleansing(self.org)
        c.cleanse(qs)

        data = c.results

        # print data
        # This only checks to make sure the 35 errors have occurred.
        self.assertEqual(len(c.results), 35)
Пример #6
0
    def test_cleanse(self):
        # Import the file and run mapping

        # Year Ending,Energy Score,Total GHG Emissions (MtCO2e),Weather Normalized Site EUI (kBtu/ft2),
        # National Median Site EUI (kBtu/ft2),Source EUI (kBtu/ft2),Weather Normalized Source EUI (kBtu/ft2),
        # National Median Source EUI (kBtu/ft2),Parking - Gross Floor Area (ft2),Organization
        # Release Date
        fake_mappings = {
            'pm_property_id': u'Property Id',
            'property_name': u'Property Name',
            'address_line_1': u'Address 1',
            'address_line_2': u'Address 2',
            'city': u'City',
            'state_province': u'State/Province',
            'postal_code': u'Postal Code',
            'year_built': u'Year Built',
            'gross_floor_area': u'Property Floor Area (Buildings and Parking) (ft2)',
            'site_eui': u'Site EUI (kBtu/ft2)',
            'generation_date': u'Generation Date'
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=PORTFOLIO_BS,
        ).iterator()

        c = Cleansing(self.org)
        c.cleanse(qs)

        data = c.results

        self.assertEqual(len(c.results), 2)

        result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{
            'field': u'pm_property_id',
            'formatted_field': u'PM Property ID',
            'value': u'',
            'message': u'PM Property ID is missing',
            'detailed_message': u'PM Property ID is missing',
            'severity': u'error'
        }]
        self.assertEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{
            'field': u'site_eui',
            'formatted_field': u'Site EUI',
            'value': 0.1,
            'message': u'Site EUI out of range',
            'detailed_message': u'Site EUI [0.1] < 10.0',
            'severity': u'warning'
        }]
        self.assertEqual(res, result['cleansing_results'])
Пример #7
0
 def handle(self, *args, **options):
     pk = int(args[0])
     print "Importing file %s" % pk
     print save_raw_data(pk)
Пример #8
0
    def test_cleanse(self):
        # Import the file and run mapping

        # This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
        # value is the value in the CSV
        fake_mappings = {
            'city': 'city',
            'postal_code': 'Zip',
            'gross_floor_area': 'GBA',
            'building_count': 'BLDGS',
            'year_built': 'AYB_YearBuilt',
            'state_province': 'State',
            'address_line_1': 'Address',
            'owner': 'Owner',
            'property_notes': 'Property Type',
            'tax_lot_id': 'UBI',
            'custom_id_1': 'Custom ID',
            'pm_property_id': 'PM Property ID'
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=ASSESSED_BS,
        ).iterator()

        c = Cleansing(self.org)
        c.cleanse(qs)

        self.assertEqual(len(c.results), 2)

        result = [v for v in c.results.values() if v['address_line_1'] == '95373 E Peach Avenue']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        self.assertTrue(result['address_line_1'], '95373 E Peach Avenue')
        self.assertTrue(result['tax_lot_id'], '10107/c6596')
        res = [{
            'field': u'pm_property_id',
            'formatted_field': u'PM Property ID',
            'value': u'',
            'message': u'PM Property ID is missing',
            'detailed_message': u'PM Property ID is missing',
            'severity': u'error'
        }]
        self.assertEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '120243 E True Lane']
        if len(result) == 1:
            result = result[0]
        else:
            raise RuntimeError('Non unity results')

        res = [{
            'field': u'year_built',
            'formatted_field': u'Year Built',
            'value': 0,
            'message': u'Year Built out of range',
            'detailed_message': u'Year Built [0] < 1700',
            'severity': u'error'
        }, {
            'field': u'gross_floor_area',
            'formatted_field': u'Gross Floor Area',
            'value': 10000000000.0,
            'message': u'Gross Floor Area out of range',
            'detailed_message': u'Gross Floor Area [10000000000.0] > 7000000.0',
            'severity': u'error'
        }, {
            'field': u'custom_id_1',
            'formatted_field': u'Custom ID 1',
            'value': u'',
            'message': u'Custom ID 1 is missing',
            'detailed_message': u'Custom ID 1 is missing',
            'severity': u'error'
        }, {
            'field': u'pm_property_id',
            'formatted_field': u'PM Property ID',
            'value': u'',
            'message': u'PM Property ID is missing',
            'detailed_message': u'PM Property ID is missing',
            'severity': u'error'
        }]
        self.assertItemsEqual(res, result['cleansing_results'])

        result = [v for v in c.results.values() if v['address_line_1'] == '1234 Peach Tree Avenue']
        self.assertEqual(len(result), 0)
        self.assertEqual(result, [])
Пример #9
0
    def test_cleanse(self):
        # Import the file and run mapping

        # This is silly, the mappings are backwards from what you would expect. The key is the BS field, and the
        # value is the value in the CSV
        fake_mappings = {
            'block_number': 'block_number',
            'error_type': 'error type',
            'building_count': 'building_count',
            'conditioned_floor_area': 'conditioned_floor_area',
            'energy_score': 'energy_score',
            'gross_floor_area': 'gross_floor_area',
            'lot_number': 'lot_number',
            'occupied_floor_area': 'occupied_floor_area',
            'postal_code': 'postal_code',
            'site_eui': 'site_eui',
            'site_eui_weather_normalized': 'site_eui_weather_normalized',
            'source_eui': 'source_eui',
            'source_eui_weather_normalized': 'source_eui_weather_normalized',
            'address_line_1': 'address_line_1',
            'address_line_2': 'address_line_2',
            'building_certification': 'building_certification',
            'city': 'city',
            'custom_id_1': 'custom_id_1',
            'district': 'district',
            'energy_alerts': 'energy_alerts',
            'owner': 'owner',
            'owner_address': 'owner_address',
            'owner_city_state': 'owner_city_state',
            'owner_email': 'owner_email',
            'owner_postal_code': 'owner_postal_code',
            'owner_telephone': 'owner_telephone',
            'pm_property_id': 'pm_property_id',
            'property_name': 'property_name',
            'property_notes': 'property_notes',
            'space_alerts': 'space_alerts',
            'state_province': 'state_province',
            'tax_lot_id': 'tax_lot_id',
            'use_description': 'use_description',
            'generation_date': 'generation_date',
            'recent_sale_date': 'recent_sale_date',
            'release_date': 'release_date',
            'year_built': 'year_built',
            'year_ending': 'year_ending',
        }

        tasks.save_raw_data(self.import_file.id)
        util.make_fake_mappings(fake_mappings, self.org)
        tasks.map_data(self.import_file.id)

        qs = BuildingSnapshot.objects.filter(
            import_file=self.import_file,
            source_type=ASSESSED_BS,
        ).iterator()

        c = Cleansing(self.org)
        c.cleanse(qs)

        # print data
        # This only checks to make sure the 35 errors have occurred.
        self.assertEqual(len(c.results), 35)