def save_column_names(model_obj): """Save unique column names for extra_data in this organization. Basically this is a record of all the extra_data keys we've ever seen for a particular organization. :param model_obj: model_obj instance (either PropertyState or TaxLotState). """ md = MappingData() for key in model_obj.extra_data: # Ascertain if our key is ``extra_data`` or not. # This is doing way to much work to find if the fields are extra data, especially # since that has been asked probably many times before. db_field = md.find_column(model_obj.__class__.__name__, key) is_extra_data = False if db_field else True # yes i am a db column, thus I am not extra_data # get the name of the model object as a string to save into the database Column.objects.get_or_create( column_name=key[:511], is_extra_data=is_extra_data, organization=model_obj.organization, table_name=model_obj.__class__.__name__ )
def save_column_names(model_obj): """Save unique column names for extra_data in this organization. This is a record of all the extra_data keys we have ever seen for a particular organization. :param model_obj: model_obj instance (either PropertyState or TaxLotState). """ md = MappingData() for key in model_obj.extra_data: # Ascertain if our key is ``extra_data`` or not. # This is doing way to much work to find if the fields are extra data, especially # since that has been asked probably many times before. db_field = md.find_column(model_obj.__class__.__name__, key) is_extra_data = False if db_field else True # yes i am a db column, thus I am not extra_data # handle the special edge-case where an old organization may have duplicate columns # in the database. We should make this a migration in the future and put a validation # in the db. for i in range(0, 5): while True: try: Column.objects.get_or_create( column_name=key[:511], is_extra_data=is_extra_data, organization=model_obj.organization, table_name=model_obj.__class__.__name__ ) except Column.MultipleObjectsReturned: _log.debug( "Column.MultipleObjectsReturned for {} in save_column_names".format( key[:511])) columns = Column.objects.filter(column_name=key[:511], is_extra_data=is_extra_data, organization=model_obj.organization, table_name=model_obj.__class__.__name__) for c in columns: if not ColumnMapping.objects.filter( Q(column_raw=c) | Q(column_mapped=c)).exists(): _log.debug("Deleting column object {}".format(c.column_name)) c.delete() # Check if there are more than one column still if Column.objects.filter( column_name=key[:511], is_extra_data=is_extra_data, organization=model_obj.organization, table_name=model_obj.__class__.__name__).count() > 1: raise Exception( "Could not fix duplicate columns for {}. Contact dev team").format( key) continue break
def _retrieve_db_columns(): """ Returns a predefined list of columns that can be in the database. This is a hardcoded list of all the database fields along with additional information such as the display name. :return: dict """ # Grab the default columns and their details hard_coded_columns = copy.deepcopy(VIEW_COLUMNS_PROPERTY) md = MappingData() for c in hard_coded_columns: if not md.find_column(c['table'], c['name']): print "Could not find column field in database for {}".format(c) return hard_coded_columns
# !/usr/bin/env python # encoding: utf-8 """ :copyright (c) 2014 - 2018, The Regents of the University of California, through Lawrence Berkeley National Laboratory (subject to receipt of any required approvals from the U.S. Department of Energy) and contributors. All rights reserved. # NOQA :author Dan Gunter <*****@*****.**> """ import logging from collections import defaultdict from seed.lib.mappings.mapping_data import MappingData from seed.models import ( PropertyState, TaxLotState, ) md = MappingData() property_state_fields = [x['name'] for x in sorted(md.property_state_data)] tax_lot_state_fields = [x['name'] for x in md.tax_lot_state_data] # TODO: Move these methods to the MappingData object and return from there PropertyState_to_PropertyState = tuple([ (k, k) for k in sorted(property_state_fields) ]) TaxLotState_to_TaxLotState = tuple([(k, k) for k in tax_lot_state_fields]) _log = logging.getLogger(__name__) def get_attrs_with_mapping(data_set_buildings, mapping): """ Returns a dictionary of attributes from each data_set_building.
def _column_fields_to_columns(fields, organization): """ List of dictionaries to process into column objects. This method will create the columns if they did not previously exist. Note that fields are probably mutable, but the method returns a new list of fields. .. example: test_map = [ { 'from_field': 'eui', 'from_units': 'kBtu/ft**2/year', # optional 'to_field': 'site_eui', 'to_table_name': 'PropertyState', }, { 'from_field': 'address', 'to_field': 'address', 'to_table_name': 'TaxLotState' }, { 'from_field': 'Wookiee', 'to_field': 'Dothraki', 'to_table_name': 'PropertyState', }, ] Args: fields: list of dicts containing to and from fields organization: organization model instance Returns: dict with lists of columns to which is mappable. """ def select_col_obj(column_name, table_name, organization_column): if organization_column: return [organization_column] else: # Try for "global" column definitions, e.g. BEDES. - Note the BEDES are not # loaded into the database as of 9/8/2016 so not sure if this code is ever # exercised obj = Column.objects.filter(organization=None, column_name=column_name).first() if obj: # create organization mapped column obj.pk = None obj.id = None obj.organization = organization obj.save() return [obj] else: if table_name: obj, _ = Column.objects.get_or_create( organization=organization, column_name=column_name, table_name=table_name, is_extra_data=is_extra_data, ) return [obj] else: obj, _ = Column.objects.get_or_create( organization=organization, column_name=column_name, is_extra_data=is_extra_data, ) return [obj] md = MappingData() # Container to store the dicts with the Column object new_data = [] for field in fields: new_field = field # find the mapping data column (i.e. the database fields) that match, if it exists # then set the extra data flag to true db_field = md.find_column(field['to_table_name'], field['to_field']) is_extra_data = False if db_field else True # yes i am a db column, thus I am not extra_data try: to_org_col, _ = Column.objects.get_or_create( organization=organization, column_name=field['to_field'], table_name=field['to_table_name'], is_extra_data=is_extra_data) except Column.MultipleObjectsReturned: _log.debug("More than one to_column found for {}.{}".format( field['to_table_name'], field['to_field'])) # raise Exception("Cannot handle more than one to_column returned for {}.{}".format( # field['to_field'], field['to_table_name'])) # TODO: write something to remove the duplicate columns to_org_col = Column.objects.filter( organization=organization, column_name=field['to_field'], table_name=field['to_table_name'], is_extra_data=is_extra_data).first() _log.debug("Grabbing the first to_column") try: # the from column is the field in the import file, thus the table_name needs to be # blank. Eventually need to handle passing in import_file_id from_org_col, _ = Column.objects.get_or_create( organization=organization, table_name__in=[None, ''], column_name=field['from_field'], units_pint=field.get('from_units'), # might be None is_extra_data= False # data from header rows in the files are NEVER extra data ) except Column.MultipleObjectsReturned: _log.debug("More than one from_column found for {}.{}".format( field['to_table_name'], field['to_field'])) # TODO: write something to remove the duplicate columns from_org_col = Column.objects.filter( organization=organization, table_name__in=[None, ''], column_name=field['from_field'], units_pint=field.get('from_units'), # might be None is_extra_data=is_extra_data).first() _log.debug("Grabbing the first from_column") new_field['to_column_object'] = select_col_obj( field['to_field'], field['to_table_name'], to_org_col) new_field['from_column_object'] = select_col_obj( field['from_field'], "", from_org_col) new_data.append(new_field) return new_data
def _column_fields_to_columns(fields, organization): """ List of dictionaries to process into column objects. This method will create the columns if they did not previously exist. Note that fields are probably mutable, but the method returns a new list of fields. .. example: test_map = [ { 'from_field': 'eui', 'to_field': 'site_eui', 'to_table_name': 'PropertyState', }, { 'from_field': 'address', 'to_field': 'address', 'to_table_name': 'TaxLotState' }, { 'from_field': 'Wookiee', 'to_field': 'Dothraki', 'to_table_name': 'PropertyState', }, ] Args: fields: list of dicts containing to and from fields organization: organization model instance Returns: dict with lists of columns to which is mappable. """ def select_col_obj(column_name, table_name, organization_column): if organization_column: return [organization_column] else: # Try for "global" column definitions, e.g. BEDES. - Note the BEDES are not # loaded into the database as of 9/8/2016 so not sure if this code is ever # exercised obj = Column.objects.filter(organization=None, column_name=column_name).first() if obj: # create organization mapped column obj.pk = None obj.id = None obj.organization = organization obj.save() return [obj] else: if table_name: obj, _ = Column.objects.get_or_create( organization=organization, column_name=column_name, table_name=table_name, is_extra_data=is_extra_data, ) return [obj] else: obj, _ = Column.objects.get_or_create( organization=organization, column_name=column_name, is_extra_data=is_extra_data, ) return [obj] return True md = MappingData() # Container to store the dicts with the Column object new_data = [] for field in fields: new_field = field # find the mapping data column (i.e. the database fields) that match, if it exists # then set the extra data flag to true db_field = md.find_column(field['to_table_name'], field['to_field']) is_extra_data = False if db_field else True # yes i am a db column, thus I am not extra_data # find the to_column to_org_col = Column.objects.filter(organization=organization, column_name=field['to_field'], table_name=field['to_table_name'], is_extra_data=is_extra_data).first() from_org_col = Column.objects.filter(organization=organization, column_name=field['from_field'], is_extra_data=is_extra_data).first() new_field['to_column_object'] = select_col_obj( field['to_field'], field['to_table_name'], to_org_col ) new_field['from_column_object'] = select_col_obj( field['from_field'], "", from_org_col) new_data.append(new_field) return new_data
def process(self, organization_id, cycle, property_view=None): """ Process the building file that was uploaded and create the correct models for the object :param organization_id: integer, ID of organization :param cycle: object, instance of cycle object :param property_view: Existing property view of the building file that will be updated from merging the property_view.state :return: list, [status, (PropertyState|None), (PropertyView|None), messages] """ Parser = self.BUILDING_FILE_PARSERS.get(self.file_type, None) if not Parser: acceptable_file_types = ', '.join( map(dict(self.BUILDING_FILE_TYPES).get, self.BUILDING_FILE_PARSERS.keys()) ) return False, None, None, "File format was not one of: {}".format(acceptable_file_types) parser = Parser() parser.import_file(self.file.path) parser_args = [] parser_kwargs = {} if self.file_type == self.BUILDINGSYNC: parser_args.append(BuildingSync.BRICR_STRUCT) data, errors, messages = parser.process(*parser_args, **parser_kwargs) if errors or not data: return False, None, None, messages # sub-select the data that are needed to create the PropertyState object md = MappingData() create_data = {"organization_id": organization_id} extra_data = {} for k, v in data.items(): # Skip the keys that are for measures and reports and process later if k in ['measures', 'reports', 'scenarios']: continue if md.find_column('PropertyState', k): create_data[k] = v else: # TODO: break out columns in the extra data that should be part of the # PropertyState and which ones should be added to some other class that # doesn't exist yet. extra_data[k] = v # create columns, if needed, for the extra_data fields Column.objects.get_or_create( organization_id=organization_id, column_name=k, table_name='PropertyState', is_extra_data=True, ) # always create the new object, then decide if we need to merge it. # create a new property_state for the object and promote to a new property_view property_state = PropertyState.objects.create(**create_data) property_state.extra_data = extra_data property_state.save() PropertyAuditLog.objects.create( organization_id=organization_id, state_id=property_state.id, name='Import Creation', description='Creation from Import file.', import_filename=self.file.path, record_type=AUDIT_IMPORT ) # set the property_state_id so that we can list the building files by properties self.property_state_id = property_state.id self.save() # add in the measures for m in data.get('measures', []): # Find the measure in the database try: measure = Measure.objects.get( category=m['category'], name=m['name'], organization_id=organization_id, ) except Measure.DoesNotExist: # TODO: Deal with it continue # Add the measure to the join table. # Need to determine what constitutes the unique measure for a property join, _ = PropertyMeasure.objects.get_or_create( property_state_id=self.property_state_id, measure_id=measure.pk, implementation_status=PropertyMeasure.str_to_impl_status(m['implementation_status']), application_scale=PropertyMeasure.str_to_application_scale( m.get('application_scale_of_application', PropertyMeasure.SCALE_ENTIRE_FACILITY) ), category_affected=PropertyMeasure.str_to_category_affected( m.get('system_category_affected', PropertyMeasure.CATEGORY_OTHER) ), recommended=m.get('recommended', 'false') == 'true', ) join.description = m.get('description') join.property_measure_name = m.get('property_measure_name') join.cost_mv = m.get('mv_cost') join.cost_total_first = m.get('measure_total_first_cost') join.cost_installation = m.get('measure_installation_cost') join.cost_material = m.get('measure_material_cost') join.cost_capital_replacement = m.get('measure_capital_replacement_cost') join.cost_residual_value = m.get('measure_residual_value') join.save() # add in scenarios for s in data.get('scenarios', []): # measures = models.ManyToManyField(PropertyMeasure) # {'reference_case': u'Baseline', 'annual_savings_site_energy': None, # 'measures': [], 'id': u'Baseline', 'name': u'Baseline'} scenario, _ = Scenario.objects.get_or_create( name=s.get('name'), property_state_id=self.property_state_id, ) scenario.description = s.get('description') scenario.annual_site_energy_savings = s.get('annual_site_energy_savings') scenario.annual_source_energy_savings = s.get('annual_source_energy_savings') scenario.annual_cost_savings = s.get('annual_cost_savings') scenario.summer_peak_load_reduction = s.get('summer_peak_load_reduction') scenario.winter_peak_load_reduction = s.get('winter_peak_load_reduction') scenario.hdd = s.get('hdd') scenario.hdd_base_temperature = s.get('hdd_base_temperature') scenario.cdd = s.get('cdd') scenario.cdd_base_temperature = s.get('cdd_base_temperature') # temporal_status = models.IntegerField(choices=TEMPORAL_STATUS_TYPES, # default=TEMPORAL_STATUS_CURRENT) if s.get('reference_case'): ref_case = Scenario.objects.filter( name=s.get('reference_case'), property_state_id=self.property_state_id, ) if len(ref_case) == 1: scenario.reference_case = ref_case.first() # set the list of measures for measure_name in s['measures']: # find the join measure in the database measure = None try: measure = PropertyMeasure.objects.get( property_state_id=self.property_state_id, property_measure_name=measure_name, ) except PropertyMeasure.DoesNotExist: # PropertyMeasure is not in database, skipping silently continue scenario.measures.add(measure) scenario.save() if property_view: # create a new blank state to merge the two together merged_state = PropertyState.objects.create(organization_id=organization_id) # assume the same cycle id as the former state. # should merge_state also copy/move over the relationships? merged_state, changed = merge_state(merged_state, property_view.state, property_state, get_state_attrs([property_view.state, property_state])) # log the merge # Not a fan of the parent1/parent2 logic here, seems error prone, what this # is also in here: https://github.com/SEED-platform/seed/blob/63536e99cf5be3a9a86391c5cead6dd4ff74462b/seed/data_importer/tasks.py#L1549 PropertyAuditLog.objects.create( organization_id=organization_id, parent1=PropertyAuditLog.objects.filter(state=property_view.state).first(), parent2=PropertyAuditLog.objects.filter(state=property_state).first(), parent_state1=property_view.state, parent_state2=property_state, state=merged_state, name='System Match', description='Automatic Merge', import_filename=None, record_type=AUDIT_IMPORT ) property_view.state = merged_state property_view.save() merged_state.merge_state = MERGE_STATE_MERGED merged_state.save() # set the property_state to the new one property_state = merged_state elif not property_view: property_view = property_state.promote(cycle) else: # invalid arguments, must pass both or neither return False, None, None, "Invalid arguments passed to BuildingFile.process()" return True, property_state, property_view, messages