def test_retrieve_all_as_tuple(self): list_result = Column.retrieve_all_by_tuple(self.fake_org) self.assertIn(('PropertyState', 'site_eui_modeled'), list_result) self.assertIn(('TaxLotState', 'tax_lot_id_not_used'), list_result) self.assertIn(('PropertyState', 'gross_floor_area'), list_result) # extra field in taxlot, but not in property self.assertIn(('TaxLotState', 'Gross Floor Area'), list_result) # extra field in taxlot, but not in property
def test_mapping(self): """Test objects in database can be converted to mapped fields""" # for mapping, you have to create an import file, even it is just one record. This is # more of an ID to track imports state = self.property_state_factory.get_property_state_as_extra_data( import_file_id=self.import_file.id, source_type=ASSESSED_RAW, data_state=DATA_STATE_IMPORT, random_extra=42) # set import_file save done to true self.import_file.raw_save_done = True self.import_file.save() # Create mappings from the new states # TODO #239: Convert this to a single helper method to suggest and save suggested_mappings = mapper.build_column_mapping( list(state.extra_data.keys()), Column.retrieve_all_by_tuple(self.org), previous_mapping=get_column_mapping, map_args=[self.org], thresh=80) # Convert mapping suggests to the format needed for saving mappings = [] for raw_column, suggestion in suggested_mappings.items(): # Single suggestion looks like:'lot_number': ['PropertyState', 'lot_number', 100] mapping = { "from_field": raw_column, "from_units": None, "to_table_name": suggestion[0], "to_field": suggestion[1], "to_field_display_name": suggestion[1], } mappings.append(mapping) # Now save the mappings # print(mappings) Column.create_mappings(mappings, self.org, self.user, self.import_file.id) # END TODO tasks.map_data(self.import_file.id) props = self.import_file.find_unmatched_property_states() self.assertEqual(len(props), 1) self.assertEqual(state.extra_data['year_built'], props.first().year_built) self.assertEqual(state.extra_data['random_extra'], props.first().extra_data['random_extra'])
def suggestions(self, request): """ Retrieves suggestions given raw column headers. parameters: - headers:--------------------------------------------------------------------------------------------------------------------------- - name: organization_id description: The organization_id for this user's organization required: true (at least, nothing will be returned if not provided) paramType: query """ try: org_id = request.query_params.get('organization_id', None) raw_headers = request.data.get('headers', []) suggested_mappings = mapper.build_column_mapping( raw_headers, Column.retrieve_all_by_tuple(org_id), previous_mapping=None, map_args=None, thresh= 80 # percentage match that we require. 80% is random value for now. ) # replace None with empty string for column names and PropertyState for tables # TODO #239: Move this fix to build_column_mapping for m in suggested_mappings: table, destination_field, _confidence = suggested_mappings[m] if destination_field is None: suggested_mappings[m][1] = '' # Fix the table name, eventually move this to the build_column_mapping for m in suggested_mappings: table, _destination_field, _confidence = suggested_mappings[m] # Do not return the campus, created, updated fields... that is force them to be in the property state if not table or table == 'Property': suggested_mappings[m][0] = 'PropertyState' elif table == 'TaxLot': suggested_mappings[m][0] = 'TaxLotState' return JsonResponse({ 'status': 'success', 'data': suggested_mappings, }) except Exception as e: return JsonResponse({ 'status': 'error', 'data': str(e), }, status=HTTP_400_BAD_REQUEST)
def test_mapping_takes_into_account_selected_units(self): # Just as in the previous test, build extra_data PropertyState raw_state = self.property_state_factory.get_property_state_as_extra_data( import_file_id=self.import_file.id, source_type=ASSESSED_RAW, data_state=DATA_STATE_IMPORT, ) # Replace the site_eui and gross_floor_area key-value that gets # autogenerated by get_property_state_as_extra_data del raw_state.extra_data['site_eui'] raw_state.extra_data['Site EUI'] = 100 del raw_state.extra_data['gross_floor_area'] raw_state.extra_data['Gross Floor Area'] = 100 raw_state.save() self.import_file.raw_save_done = True self.import_file.save() # Build mappings - with unit-aware destinations and non-default unit choices suggested_mappings = mapper.build_column_mapping( list(raw_state.extra_data.keys()), Column.retrieve_all_by_tuple(self.org), previous_mapping=get_column_mapping, map_args=[self.org], thresh=80) mappings = [] for raw_column, suggestion in suggested_mappings.items(): if raw_column == 'Site EUI': mappings.append({ "from_field": raw_column, "from_units": 'kWh/m**2/year', "to_table_name": 'PropertyState', "to_field": 'site_eui', "to_field_display_name": 'Site EUI', }) elif raw_column == 'Gross Floor Area': mappings.append({ "from_field": raw_column, "from_units": 'm**2', "to_table_name": 'PropertyState', "to_field": 'gross_floor_area', "to_field_display_name": 'Gross Floor Area', }) else: other_mapping = { "from_field": raw_column, "from_units": None, "to_table_name": suggestion[0], "to_field": suggestion[1], "to_field_display_name": suggestion[1], } mappings.append(other_mapping) # Perform mapping, creating the initial PropertyState records. Column.create_mappings(mappings, self.org, self.user, self.import_file.id) tasks.map_data(self.import_file.id) # Verify that the values have been converted appropriately state = self.import_file.find_unmatched_property_states().get() self.assertAlmostEqual(state.site_eui, (100 * ureg('kWh/m**2/year')).to('kBtu/ft**2/year')) self.assertAlmostEqual(state.gross_floor_area, (100 * ureg('m**2')).to('ft**2'))
def test_remapping_with_and_without_unit_aware_columns_doesnt_lose_data( self): """ During import, when the initial -State objects are created from the extra_data values, ColumnMapping objects are used to take the extra_data dictionary values and create the -State objects, setting the DB-level values as necessary - e.g. taking a raw "Site EUI (kBtu/ft2)" value and inserting it into the DB field "site_eui". Previously, remapping could cause extra Column objects to be created, and subsequently, this created extra ColumnMapping objects. These extra ColumnMapping objects could cause raw values to be inserted into the wrong DB field on -State creation. """ # Just as in the previous test, build extra_data PropertyState state = self.property_state_factory.get_property_state_as_extra_data( import_file_id=self.import_file.id, source_type=ASSESSED_RAW, data_state=DATA_STATE_IMPORT, random_extra=42, ) # Replace the site_eui key-value that gets autogenerated by get_property_state_as_extra_data del state.extra_data['site_eui'] state.extra_data['Site EUI (kBtu/ft2)'] = 123 state.save() self.import_file.raw_save_done = True self.import_file.save() # Build 2 sets of mappings - with and without a unit-aware destination site_eui data suggested_mappings = mapper.build_column_mapping( list(state.extra_data.keys()), Column.retrieve_all_by_tuple(self.org), previous_mapping=get_column_mapping, map_args=[self.org], thresh=80) ed_site_eui_mappings = [] unit_aware_site_eui_mappings = [] for raw_column, suggestion in suggested_mappings.items(): if raw_column == 'Site EUI (kBtu/ft2)': # Make this an extra_data field (without from_units) ed_site_eui_mappings.append({ "from_field": raw_column, "from_units": None, "to_table_name": 'PropertyState', "to_field": raw_column, "to_field_display_name": raw_column, }) unit_aware_site_eui_mappings.append({ "from_field": raw_column, "from_units": 'kBtu/ft**2/year', "to_table_name": 'PropertyState', "to_field": 'site_eui', "to_field_display_name": 'Site EUI', }) else: other_mapping = { "from_field": raw_column, "from_units": None, "to_table_name": suggestion[0], "to_field": suggestion[1], "to_field_display_name": suggestion[1], } ed_site_eui_mappings.append(other_mapping) unit_aware_site_eui_mappings.append(other_mapping) # Map and remap the file multiple times with different mappings each time. # Round 1 - Map site_eui data into Extra Data Column.create_mappings(ed_site_eui_mappings, self.org, self.user, self.import_file.id) tasks.map_data(self.import_file.id) # There should only be one raw 'Site EUI (kBtu/ft2)' Column object self.assertEqual( 1, self.org.column_set.filter(column_name='Site EUI (kBtu/ft2)', table_name='').count()) # The one propertystate should have site eui info in extra_data prop = self.import_file.find_unmatched_property_states().get() self.assertIsNone(prop.site_eui) self.assertIsNotNone(prop.extra_data.get('Site EUI (kBtu/ft2)')) # Round 2 - Map site_eui data into the PropertyState attribute "site_eui" Column.create_mappings(unit_aware_site_eui_mappings, self.org, self.user, self.import_file.id) tasks.map_data(self.import_file.id, remap=True) self.assertEqual( 1, self.org.column_set.filter(column_name='Site EUI (kBtu/ft2)', table_name='').count()) # The one propertystate should have site eui info in site_eui prop = self.import_file.find_unmatched_property_states().get() self.assertIsNotNone(prop.site_eui) self.assertIsNone(prop.extra_data.get('Site EUI (kBtu/ft2)')) # Round 3 - Map site_eui data into Extra Data Column.create_mappings(ed_site_eui_mappings, self.org, self.user, self.import_file.id) tasks.map_data(self.import_file.id, remap=True) self.assertEqual( 1, self.org.column_set.filter(column_name='Site EUI (kBtu/ft2)', table_name='').count()) # The one propertystate should have site eui info in extra_data prop = self.import_file.find_unmatched_property_states().get() self.assertIsNone(prop.site_eui) self.assertIsNotNone(prop.extra_data.get('Site EUI (kBtu/ft2)'))
def mapping_suggestions(self, request, pk): """ Returns suggested mappings from an uploaded file's headers to known data fields. """ organization_id = request.query_params.get('organization_id', None) result = {'status': 'success'} membership = OrganizationUser.objects.select_related('organization') \ .get(organization_id=organization_id, user=request.user) organization = membership.organization # For now, each organization holds their own mappings. This is non-ideal, but it is the # way it is for now. In order to move to parent_org holding, then we need to be able to # dynamically match columns based on the names and not the db id (or support many-to-many). # parent_org = organization.get_parent() try: import_file = ImportFile.objects.get( pk=pk, import_record__super_organization_id=organization.pk) except ImportFile.DoesNotExist: return JsonResponse( { 'status': 'error', 'message': 'Could not find import file with pk=' + str(pk) }, status=status.HTTP_400_BAD_REQUEST) # Get a list of the database fields in a list, these are the db columns and the extra_data columns property_columns = Column.retrieve_mapping_columns( organization.pk, 'property') taxlot_columns = Column.retrieve_mapping_columns( organization.pk, 'taxlot') # If this is a portfolio manager file, then load in the PM mappings and if the column_mappings # are not in the original mappings, default to PM if import_file.from_portfolio_manager: pm_mappings = simple_mapper.get_pm_mapping( import_file.first_row_columns, resolve_duplicates=True) suggested_mappings = mapper.build_column_mapping( import_file.first_row_columns, Column.retrieve_all_by_tuple(organization_id), previous_mapping=get_column_mapping, map_args=[organization], default_mappings=pm_mappings, thresh=80) elif import_file.from_buildingsync: bsync_mappings = xml_mapper.build_column_mapping() suggested_mappings = mapper.build_column_mapping( import_file.first_row_columns, Column.retrieve_all_by_tuple(organization_id), previous_mapping=get_column_mapping, map_args=[organization], default_mappings=bsync_mappings, thresh=80) else: # All other input types suggested_mappings = mapper.build_column_mapping( import_file.first_row_columns, Column.retrieve_all_by_tuple(organization.pk), previous_mapping=get_column_mapping, map_args=[organization], thresh= 80 # percentage match that we require. 80% is random value for now. ) # replace None with empty string for column names and PropertyState for tables # TODO #239: Move this fix to build_column_mapping for m in suggested_mappings: table, destination_field, _confidence = suggested_mappings[m] if destination_field is None: suggested_mappings[m][1] = '' # Fix the table name, eventually move this to the build_column_mapping for m in suggested_mappings: table, _destination_field, _confidence = suggested_mappings[m] # Do not return the campus, created, updated fields... that is force them to be in the property state if not table or table == 'Property': suggested_mappings[m][0] = 'PropertyState' elif table == 'TaxLot': suggested_mappings[m][0] = 'TaxLotState' result['suggested_column_mappings'] = suggested_mappings result['property_columns'] = property_columns result['taxlot_columns'] = taxlot_columns return JsonResponse(result)