def test_snake_case(self): self.assertEqual(_snake_case("AbCdEf"), "ab_cd_ef") self.assertEqual(_snake_case("Clean and/or repair"), "clean_and_or_repair") self.assertEqual( _snake_case( "Upgrade operating protocols, calibration, and/or sequencing"), "upgrade_operating_protocols_calibration_and_or_sequencing") self.assertEqual(_snake_case("AdvancedMeteringSystems"), "advanced_metering_systems")
def _process_struct(self, struct, data): """ Take a dictionary and return the `return` object with values filled in. :param struct: dict, object to parse and fill from BuildingSync file :param data: dict, data to parse and fill :return: list, the `return` value, if all paths were found, and list of messages """ def _lookup_sub(node, key_path_name, key_path_value, value_path_name): items = [node] if isinstance(node, dict) else node for item in items: found = False for k, v in item.items(): if k == key_path_name and v == key_path_value: found = True if found and k == value_path_name: return v res = {'measures': [], 'scenarios': []} messages = {'errors': [], 'warnings': []} for k, v in struct['return'].items(): path = ".".join([struct['root'], v['path']]) value = self._get_node(path, data, []) try: if v.get('key_path_name', None) and v.get('value_path_name', None) and v.get('key_path_value', None): value = _lookup_sub( value, v.get('key_path_name'), v.get('key_path_value'), v.get('value_path_name'), ) # check if the value is not defined and if it is required if not value: if v.get('required'): messages['errors'].append( "Could not find required value for sub-lookup of {}:{}".format( v.get('key_path_name'), v.get('key_path_value'))) continue else: continue if value: # catch some errors if isinstance(value, list): messages['errors'].append( "Could not find single entry for '{}'".format(path) ) continue # type cast the value if v['type'] == 'double': value = float(value) elif v['type'] == 'integer': value = int(value) elif v['type'] == 'dict': value = dict(value) elif v['type'] == 'string': value = str(value) else: messages['errors'].append( "Unknown cast type of {} for '{}'".format(v['type'], path) ) res[k] = value else: if v['required']: messages['errors'].append( "Could not find required value for '{}'".format(path) ) except Exception as err: message = "Error processing {}:{} with error: {}".format(k, v, err) messages['errors'].append(message) # manually add in parsing of measures and reports because they are a bit different than # a straight mapping # <auc:Measure ID="Measure-70165601915860"> # <auc:SystemCategoryAffected>Plug Load</auc:SystemCategoryAffected> # <auc:TechnologyCategories> # <auc:TechnologyCategory> # <auc:PlugLoadReductions> # <auc:MeasureName>Replace with ENERGY STAR rated</auc:MeasureName> # </auc:PlugLoadReductions> # </auc:TechnologyCategory> # </auc:TechnologyCategories> # <auc:LongDescription>Replace with ENERGY STAR rated</auc:LongDescription> # <auc:MVCost>0.0</auc:MVCost> # <auc:UsefulLife>20.0</auc:UsefulLife> # <auc:MeasureTotalFirstCost>5499.0</auc:MeasureTotalFirstCost> # <auc:MeasureInstallationCost>0.0</auc:MeasureInstallationCost> # <auc:MeasureMaterialCost>0.0</auc:MeasureMaterialCost> # </auc:Measure> measures = self._get_node('auc:Audits.auc:Audit.auc:Measures.auc:Measure', data, []) for m in measures: if m.get('auc:TechnologyCategories', None): cat_w_namespace = list(m['auc:TechnologyCategories']['auc:TechnologyCategory'].keys())[0] category = cat_w_namespace.replace('auc:', '') new_data = { 'property_measure_name': m.get('@ID'), # This will be the IDref from the scenarios 'category': _snake_case(category), 'name': m['auc:TechnologyCategories']['auc:TechnologyCategory'][cat_w_namespace][ 'auc:MeasureName'] } for k, v in m.items(): if k in ['@ID', 'auc:PremisesAffected', 'auc:TechnologyCategories']: continue new_data[_snake_case(k.replace('auc:', ''))] = v # fix the names of the measures for "easier" look up... doing in separate step to # fit in single line. Cleanup -- when? new_data['name'] = _snake_case(new_data['name']) res['measures'].append(new_data) else: message = "Skipping measure %s due to missing TechnologyCategory" % m.get("@ID") messages['warnings'].append(message) # <auc:Scenario> # <auc:ScenarioName>Lighting Only</auc:ScenarioName> # <auc:ScenarioType> # <auc:PackageOfMeasures> # <auc:ReferenceCase IDref="Baseline"/> # <auc:MeasureIDs> # <auc:MeasureID IDref="Measure1"/> # </auc:MeasureIDs> # <auc:AnnualSavingsSiteEnergy>162654.89601696888</auc:AnnualSavingsSiteEnergy> # </auc:PackageOfMeasures> # </auc:ScenarioType> # </auc:Scenario> scenarios = self._get_node('auc:Audits.auc:Audit.auc:Report.auc:Scenarios.auc:Scenario', data, []) for s in scenarios: new_data = { 'id': s.get('@ID'), 'name': s.get('auc:ScenarioName'), } if s.get('auc:ScenarioType'): node = s['auc:ScenarioType'].get('auc:PackageOfMeasures') if node: ref_case = self._get_node('auc:ReferenceCase', node, []) if ref_case and ref_case.get('@IDref'): new_data['reference_case'] = ref_case.get('@IDref') new_data['annual_savings_site_energy'] = node.get('auc:AnnualSavingsSiteEnergy') new_data['measures'] = [] measures = self._get_node('auc:MeasureIDs.auc:MeasureID', node, []) if isinstance(measures, list): for measure in measures: if measure.get('@IDref', None): new_data['measures'].append(measure.get('@IDref')) else: if isinstance(measures, basestring): # the measure is there, but it does not have an idref continue else: if measures.get('@IDref', None): new_data['measures'].append(measures.get('@IDref')) res['scenarios'].append(new_data) return res, messages
def _process_struct(self, struct, data): """ Take a dictionary and return the `return` object with values filled in. :param struct: dict, object to parse and fill from BuildingSync file :param data: dict, data to parse and fill :return: list, the `return` value, if all paths were found, and list of messages """ def _lookup_sub(node, key_path_name, key_path_value, value_path_name): items = [node] if isinstance(node, dict) else node for item in items: found = False for k, v in item.items(): if k == key_path_name and v == key_path_value: found = True if found and k == value_path_name: return v res = {'measures': [], 'scenarios': []} messages = {'errors': [], 'warnings': []} for k, v in struct['return'].items(): path = ".".join([struct['root'], v['path']]) value = self._get_node(path, data, []) try: if v.get('key_path_name', None) and v.get( 'value_path_name', None) and v.get( 'key_path_value', None): value = _lookup_sub( value, v.get('key_path_name'), v.get('key_path_value'), v.get('value_path_name'), ) # check if the value is not defined and if it is required if not value: if v.get('required'): messages['errors'].append( "Could not find required value for sub-lookup of {}:{}" .format(v.get('key_path_name'), v.get('key_path_value'))) continue else: continue if value: # catch some errors if isinstance(value, list): messages['errors'].append( "Could not find single entry for '{}'".format( path)) continue # type cast the value if v['type'] == 'double': value = float(value) elif v['type'] == 'integer': value = int(value) elif v['type'] == 'dict': value = dict(value) elif v['type'] == 'string': value = str(value) else: messages['errors'].append( "Unknown cast type of {} for '{}'".format( v['type'], path)) res[k] = value else: if v['required']: messages['errors'].append( "Could not find required value for '{}'".format( path)) except Exception as err: message = "Error processing {}:{} with error: {}".format( k, v, err) messages['errors'].append(message) # manually add in parsing of measures and reports because they are a bit different than # a straight mapping # <auc:Measure ID="Measure-70165601915860"> # <auc:SystemCategoryAffected>Plug Load</auc:SystemCategoryAffected> # <auc:TechnologyCategories> # <auc:TechnologyCategory> # <auc:PlugLoadReductions> # <auc:MeasureName>Replace with ENERGY STAR rated</auc:MeasureName> # </auc:PlugLoadReductions> # </auc:TechnologyCategory> # </auc:TechnologyCategories> # <auc:LongDescription>Replace with ENERGY STAR rated</auc:LongDescription> # <auc:MVCost>0.0</auc:MVCost> # <auc:UsefulLife>20.0</auc:UsefulLife> # <auc:MeasureTotalFirstCost>5499.0</auc:MeasureTotalFirstCost> # <auc:MeasureInstallationCost>0.0</auc:MeasureInstallationCost> # <auc:MeasureMaterialCost>0.0</auc:MeasureMaterialCost> # </auc:Measure> measures = self._get_node( 'auc:BuildingSync.auc:Facilities.auc:Facility.auc:Measures.auc:Measure', data, []) # check that this is a list, if not, make it a list or the loop won't work correctly if isinstance(measures, dict): # print("measures is a dict...converting it to a list") measures_tmp = [] measures_tmp.append(measures) measures = measures_tmp for m in measures: if m.get('auc:TechnologyCategories', None): cat_w_namespace = list(m['auc:TechnologyCategories'] ['auc:TechnologyCategory'].keys())[0] category = cat_w_namespace.replace('auc:', '') new_data = { 'property_measure_name': m.get('@ID'), # This will be the IDref from the scenarios 'category': _snake_case(category), 'name': m['auc:TechnologyCategories']['auc:TechnologyCategory'] [cat_w_namespace]['auc:MeasureName'] } for k, v in m.items(): if k in [ '@ID', 'auc:PremisesAffected', 'auc:TechnologyCategories' ]: continue new_data[_snake_case(k.replace('auc:', ''))] = v # fix the names of the measures for "easier" look up... doing in separate step to # fit in single line. Cleanup -- when? new_data['name'] = _snake_case(new_data['name']) res['measures'].append(new_data) else: message = "Skipping measure %s due to missing TechnologyCategory" % m.get( "@ID") messages['warnings'].append(message) # <auc:Scenario> # <auc:ScenarioName>Lighting Only</auc:ScenarioName> # <auc:ScenarioType> # <auc:PackageOfMeasures> # <auc:ReferenceCase IDref="Baseline"/> # <auc:MeasureIDs> # <auc:MeasureID IDref="Measure1"/> # </auc:MeasureIDs> # <auc:AnnualSavingsSiteEnergy>162654.89601696888</auc:AnnualSavingsSiteEnergy> # </auc:PackageOfMeasures> # </auc:ScenarioType> # </auc:Scenario> # KAF: for now, handle both Reports.Report and Report scenarios = self._get_node( 'auc:BuildingSync.auc:Facilities.auc:Facility.auc:Reports.auc:Report.auc:Scenarios.auc:Scenario', data, []) if not scenarios: scenarios = self._get_node( 'auc:BuildingSync.auc:Facilities.auc:Facility.auc:Report.auc:Scenarios.auc:Scenario', data, []) # check that this is a list; if not, make it a list or the loop won't work correctly if isinstance(scenarios, dict): # print("scenarios is a dict (only found 1...converting it to a list)") scenarios_tmp = [] scenarios_tmp.append(scenarios) scenarios = scenarios_tmp for s in scenarios: new_data = { 'id': s.get('@ID'), 'name': s.get('auc:ScenarioName'), } if s.get('auc:ScenarioType'): node = s['auc:ScenarioType'].get('auc:PackageOfMeasures') if node: ref_case = self._get_node('auc:ReferenceCase', node, []) if ref_case and ref_case.get('@IDref'): new_data['reference_case'] = ref_case.get('@IDref') # fixed naming of existing scenario fields new_data['annual_site_energy_savings'] = node.get( 'auc:AnnualSavingsSiteEnergy') new_data['annual_source_energy_savings'] = node.get( 'auc:AnnualSavingsSourceEnergy') new_data['annual_cost_savings'] = node.get( 'auc:AnnualSavingsCost') # new scenario fields fuel_savings = node.get('auc:AnnualSavingsByFuels') if fuel_savings: fuel_nodes = fuel_savings.get( 'auc:AnnualSavingsByFuel') if isinstance(fuel_nodes, dict): fuel_savings_arr = [] fuel_savings_arr.append(fuel_nodes) fuel_nodes = fuel_savings_arr for f in fuel_nodes: if f.get('auc:EnergyResource') == 'Electricity': new_data['annual_electricity_savings'] = f.get( 'auc:AnnualSavingsNativeUnits') # print("ELECTRICITY: {}".format(new_data['annual_electricity_savings'])) elif f.get('auc:EnergyResource') == 'Natural gas': new_data['annual_natural_gas_savings'] = f.get( 'auc:AnnualSavingsNativeUnits') # print("GAS: {}".format(new_data['annual_natural_gas_savings'])) all_resources = s.get('auc:AllResourceTotals') if all_resources: resource_nodes = all_resources.get( 'auc:AllResourceTotal') # print("ANNUAL ENERGY: {}".format(resource_nodes)) # make it an array if isinstance(resource_nodes, dict): resource_nodes_arr = [] resource_nodes_arr.append(resource_nodes) resource_nodes = resource_nodes_arr for rn in resource_nodes: if rn.get('auc:EndUse') == 'All end uses': new_data['annual_site_energy'] = rn.get( 'auc:SiteEnergyUse') new_data[ 'annual_site_energy_use_intensity'] = rn.get( 'auc:SiteEnergyUseIntensity') new_data['annual_source_energy'] = rn.get( 'auc:SourceEnergyUse') new_data[ 'annual_source_energy_use_intensity'] = rn.get( 'auc:SourceEnergyUseIntensity') resources = [] resource_uses = s.get('auc:ResourceUses') if resource_uses: ru_nodes = resource_uses.get('auc:ResourceUse') # print("ResourceUse: {}".format(ru_nodes)) if isinstance(ru_nodes, dict): ru_nodes_arr = [] ru_nodes_arr.append(ru_nodes) ru_nodes = ru_nodes_arr for ru in ru_nodes: # store resourceID and EnergyResource -- needed for TimeSeries r = {} r['id'] = ru.get('@ID') r['type'] = ru.get('auc:EnergyResource') r['units'] = ru.get('auc:ResourceUnits') resources.append(r) # just do these 2 types for now if ru.get('auc:EnergyResource') == 'Electricity': new_data['annual_electricity_energy'] = ru.get( 'auc:AnnualFuelUseConsistentUnits' ) # in MMBtu # get demand as well new_data['annual_peak_demand'] = ru.get( 'auc:AnnualPeakConsistentUnits') # in KW elif ru.get('auc:EnergyResource') == 'Natural gas': new_data['annual_natural_gas_energy'] = ru.get( 'auc:AnnualFuelUseConsistentUnits' ) # in MMBtu # timeseries timeseriesdata = s.get('auc:TimeSeriesData') # need to know if CalculationMethod is modeled (for meters) isVirtual = False calcMethod = node.get('auc:CalculationMethod') if calcMethod is not None: isModeled = calcMethod.get('auc:Modeled') if isModeled is not None: isVirtual = True if timeseriesdata: timeseries = timeseriesdata.get('auc:TimeSeries') if isinstance(timeseries, dict): ts_nodes_arr = [] ts_nodes_arr.append(timeseries) timeseries = ts_nodes_arr new_data['meters'] = [] for ts in timeseries: source_id = ts.get('auc:ResourceUseID').get( '@IDref') # print("SOURCE ID: {}".format(source_id)) source_unit = next((item for item in resources if item['id'] == source_id), None) source_unit = source_unit[ 'units'] if source_unit is not None else None match = next((item for item in new_data['meters'] if item['source_id'] == source_id), None) if match is None: # this source_id is not yet in meters, add it meter = {} meter['source_id'] = source_id source = next((item for item in Meter.SOURCES if item[1] == 'BuildingSync'), None) meter['source'] = source[0] # for BuildingSync meter['is_virtual'] = isVirtual typeMatch = next((item for item in resources if item['id'] == source_id), None) typeMatch = typeMatch['type'].title( ) if typeMatch is not None else None # print("TYPE MATCH: {}".format(type_match)) # For "Electricity", match on 'Electric - Grid' tmp_type = "Electric - Grid" if typeMatch == 'Electricity' else typeMatch theType = next((item for item in Meter.ENERGY_TYPES if item[1] == tmp_type), None) # print("the type: {}".format(the_type)) theType = theType[ 0] if theType is not None else None meter['type'] = theType meter['readings'] = [] new_data['meters'].append(meter) # add reading connected to meter (use resourceID/source_id for matching) reading = {} # ignoring timezones...pretending all is in UTC for DB and Excel export reading['start_time'] = pytz.utc.localize( datetime.strptime(ts.get('auc:StartTimeStamp'), "%Y-%m-%dT%H:%M:%S")) reading['end_time'] = pytz.utc.localize( datetime.strptime(ts.get('auc:EndTimeStamp'), "%Y-%m-%dT%H:%M:%S")) reading['reading'] = ts.get('auc:IntervalReading') reading['source_id'] = source_id reading['source_unit'] = source_unit # append to appropriate meter (or don't import) the_meter = next( (item for item in new_data['meters'] if item['source_id'] == source_id), None) if the_meter is not None: the_meter['readings'].append(reading) # print("METERS: {}".format(new_data['meters'])) # measures new_data['measures'] = [] measures = self._get_node('auc:MeasureIDs.auc:MeasureID', node, []) if isinstance(measures, list): for measure in measures: if measure.get('@IDref', None): new_data['measures'].append( measure.get('@IDref')) else: if isinstance(measures, basestring): # the measure is there, but it does not have an idref continue else: if measures.get('@IDref', None): new_data['measures'].append( measures.get('@IDref')) res['scenarios'].append(new_data) # print("SCENARIOS: {}".format(res['scenarios'])) return res, messages
def snake_case(value): # avoid circular dependency from seed.models.measures import _snake_case return _snake_case(value)