def test_snake_case(self):
self.assertEqual(_snake_case("AbCdEf"), "ab_cd_ef")
self.assertEqual(_snake_case("Clean and/or repair"),
"clean_and_or_repair")
self.assertEqual(
_snake_case(
"Upgrade operating protocols, calibration, and/or sequencing"),
"upgrade_operating_protocols_calibration_and_or_sequencing")
self.assertEqual(_snake_case("AdvancedMeteringSystems"),
"advanced_metering_systems")
def _process_struct(self, struct, data):
"""
Take a dictionary and return the `return` object with values filled in.
:param struct: dict, object to parse and fill from BuildingSync file
:param data: dict, data to parse and fill
:return: list, the `return` value, if all paths were found, and list of messages
"""
def _lookup_sub(node, key_path_name, key_path_value, value_path_name):
items = [node] if isinstance(node, dict) else node
for item in items:
found = False
for k, v in item.items():
if k == key_path_name and v == key_path_value:
found = True
if found and k == value_path_name:
return v
res = {'measures': [], 'scenarios': []}
messages = {'errors': [], 'warnings': []}
for k, v in struct['return'].items():
path = ".".join([struct['root'], v['path']])
value = self._get_node(path, data, [])
try:
if v.get('key_path_name', None) and v.get('value_path_name', None) and v.get('key_path_value', None):
value = _lookup_sub(
value,
v.get('key_path_name'),
v.get('key_path_value'),
v.get('value_path_name'),
)
# check if the value is not defined and if it is required
if not value:
if v.get('required'):
messages['errors'].append(
"Could not find required value for sub-lookup of {}:{}".format(
v.get('key_path_name'), v.get('key_path_value')))
continue
else:
continue
if value:
# catch some errors
if isinstance(value, list):
messages['errors'].append(
"Could not find single entry for '{}'".format(path)
)
continue
# type cast the value
if v['type'] == 'double':
value = float(value)
elif v['type'] == 'integer':
value = int(value)
elif v['type'] == 'dict':
value = dict(value)
elif v['type'] == 'string':
value = str(value)
else:
messages['errors'].append(
"Unknown cast type of {} for '{}'".format(v['type'], path)
)
res[k] = value
else:
if v['required']:
messages['errors'].append(
"Could not find required value for '{}'".format(path)
)
except Exception as err:
message = "Error processing {}:{} with error: {}".format(k, v, err)
messages['errors'].append(message)
# manually add in parsing of measures and reports because they are a bit different than
# a straight mapping
# <auc:Measure ID="Measure-70165601915860">
# <auc:SystemCategoryAffected>Plug Load</auc:SystemCategoryAffected>
# <auc:TechnologyCategories>
# <auc:TechnologyCategory>
# <auc:PlugLoadReductions>
# <auc:MeasureName>Replace with ENERGY STAR rated</auc:MeasureName>
# </auc:PlugLoadReductions>
# </auc:TechnologyCategory>
# </auc:TechnologyCategories>
# <auc:LongDescription>Replace with ENERGY STAR rated</auc:LongDescription>
# <auc:MVCost>0.0</auc:MVCost>
# <auc:UsefulLife>20.0</auc:UsefulLife>
# <auc:MeasureTotalFirstCost>5499.0</auc:MeasureTotalFirstCost>
# <auc:MeasureInstallationCost>0.0</auc:MeasureInstallationCost>
# <auc:MeasureMaterialCost>0.0</auc:MeasureMaterialCost>
# </auc:Measure>
measures = self._get_node('auc:Audits.auc:Audit.auc:Measures.auc:Measure', data, [])
for m in measures:
if m.get('auc:TechnologyCategories', None):
cat_w_namespace = list(m['auc:TechnologyCategories']['auc:TechnologyCategory'].keys())[0]
category = cat_w_namespace.replace('auc:', '')
new_data = {
'property_measure_name': m.get('@ID'),
# This will be the IDref from the scenarios
'category': _snake_case(category),
'name':
m['auc:TechnologyCategories']['auc:TechnologyCategory'][cat_w_namespace][
'auc:MeasureName']
}
for k, v in m.items():
if k in ['@ID', 'auc:PremisesAffected', 'auc:TechnologyCategories']:
continue
new_data[_snake_case(k.replace('auc:', ''))] = v
# fix the names of the measures for "easier" look up... doing in separate step to
# fit in single line. Cleanup -- when?
new_data['name'] = _snake_case(new_data['name'])
res['measures'].append(new_data)
else:
message = "Skipping measure %s due to missing TechnologyCategory" % m.get("@ID")
messages['warnings'].append(message)
# <auc:Scenario>
# <auc:ScenarioName>Lighting Only</auc:ScenarioName>
# <auc:ScenarioType>
# <auc:PackageOfMeasures>
# <auc:ReferenceCase IDref="Baseline"/>
# <auc:MeasureIDs>
# <auc:MeasureID IDref="Measure1"/>
# </auc:MeasureIDs>
# <auc:AnnualSavingsSiteEnergy>162654.89601696888</auc:AnnualSavingsSiteEnergy>
# </auc:PackageOfMeasures>
# </auc:ScenarioType>
# </auc:Scenario>
scenarios = self._get_node('auc:Audits.auc:Audit.auc:Report.auc:Scenarios.auc:Scenario',
data, [])
for s in scenarios:
new_data = {
'id': s.get('@ID'),
'name': s.get('auc:ScenarioName'),
}
if s.get('auc:ScenarioType'):
node = s['auc:ScenarioType'].get('auc:PackageOfMeasures')
if node:
ref_case = self._get_node('auc:ReferenceCase', node, [])
if ref_case and ref_case.get('@IDref'):
new_data['reference_case'] = ref_case.get('@IDref')
new_data['annual_savings_site_energy'] = node.get('auc:AnnualSavingsSiteEnergy')
new_data['measures'] = []
measures = self._get_node('auc:MeasureIDs.auc:MeasureID', node, [])
if isinstance(measures, list):
for measure in measures:
if measure.get('@IDref', None):
new_data['measures'].append(measure.get('@IDref'))
else:
if isinstance(measures, basestring):
# the measure is there, but it does not have an idref
continue
else:
if measures.get('@IDref', None):
new_data['measures'].append(measures.get('@IDref'))
res['scenarios'].append(new_data)
return res, messages
def _process_struct(self, struct, data):
"""
Take a dictionary and return the `return` object with values filled in.
:param struct: dict, object to parse and fill from BuildingSync file
:param data: dict, data to parse and fill
:return: list, the `return` value, if all paths were found, and list of messages
"""
def _lookup_sub(node, key_path_name, key_path_value, value_path_name):
items = [node] if isinstance(node, dict) else node
for item in items:
found = False
for k, v in item.items():
if k == key_path_name and v == key_path_value:
found = True
if found and k == value_path_name:
return v
res = {'measures': [], 'scenarios': []}
messages = {'errors': [], 'warnings': []}
for k, v in struct['return'].items():
path = ".".join([struct['root'], v['path']])
value = self._get_node(path, data, [])
try:
if v.get('key_path_name', None) and v.get(
'value_path_name', None) and v.get(
'key_path_value', None):
value = _lookup_sub(
value,
v.get('key_path_name'),
v.get('key_path_value'),
v.get('value_path_name'),
)
# check if the value is not defined and if it is required
if not value:
if v.get('required'):
messages['errors'].append(
"Could not find required value for sub-lookup of {}:{}"
.format(v.get('key_path_name'),
v.get('key_path_value')))
continue
else:
continue
if value:
# catch some errors
if isinstance(value, list):
messages['errors'].append(
"Could not find single entry for '{}'".format(
path))
continue
# type cast the value
if v['type'] == 'double':
value = float(value)
elif v['type'] == 'integer':
value = int(value)
elif v['type'] == 'dict':
value = dict(value)
elif v['type'] == 'string':
value = str(value)
else:
messages['errors'].append(
"Unknown cast type of {} for '{}'".format(
v['type'], path))
res[k] = value
else:
if v['required']:
messages['errors'].append(
"Could not find required value for '{}'".format(
path))
except Exception as err:
message = "Error processing {}:{} with error: {}".format(
k, v, err)
messages['errors'].append(message)
# manually add in parsing of measures and reports because they are a bit different than
# a straight mapping
# <auc:Measure ID="Measure-70165601915860">
# <auc:SystemCategoryAffected>Plug Load</auc:SystemCategoryAffected>
# <auc:TechnologyCategories>
# <auc:TechnologyCategory>
# <auc:PlugLoadReductions>
# <auc:MeasureName>Replace with ENERGY STAR rated</auc:MeasureName>
# </auc:PlugLoadReductions>
# </auc:TechnologyCategory>
# </auc:TechnologyCategories>
# <auc:LongDescription>Replace with ENERGY STAR rated</auc:LongDescription>
# <auc:MVCost>0.0</auc:MVCost>
# <auc:UsefulLife>20.0</auc:UsefulLife>
# <auc:MeasureTotalFirstCost>5499.0</auc:MeasureTotalFirstCost>
# <auc:MeasureInstallationCost>0.0</auc:MeasureInstallationCost>
# <auc:MeasureMaterialCost>0.0</auc:MeasureMaterialCost>
# </auc:Measure>
measures = self._get_node(
'auc:BuildingSync.auc:Facilities.auc:Facility.auc:Measures.auc:Measure',
data, [])
# check that this is a list, if not, make it a list or the loop won't work correctly
if isinstance(measures, dict):
# print("measures is a dict...converting it to a list")
measures_tmp = []
measures_tmp.append(measures)
measures = measures_tmp
for m in measures:
if m.get('auc:TechnologyCategories', None):
cat_w_namespace = list(m['auc:TechnologyCategories']
['auc:TechnologyCategory'].keys())[0]
category = cat_w_namespace.replace('auc:', '')
new_data = {
'property_measure_name':
m.get('@ID'),
# This will be the IDref from the scenarios
'category':
_snake_case(category),
'name':
m['auc:TechnologyCategories']['auc:TechnologyCategory']
[cat_w_namespace]['auc:MeasureName']
}
for k, v in m.items():
if k in [
'@ID', 'auc:PremisesAffected',
'auc:TechnologyCategories'
]:
continue
new_data[_snake_case(k.replace('auc:', ''))] = v
# fix the names of the measures for "easier" look up... doing in separate step to
# fit in single line. Cleanup -- when?
new_data['name'] = _snake_case(new_data['name'])
res['measures'].append(new_data)
else:
message = "Skipping measure %s due to missing TechnologyCategory" % m.get(
"@ID")
messages['warnings'].append(message)
# <auc:Scenario>
# <auc:ScenarioName>Lighting Only</auc:ScenarioName>
# <auc:ScenarioType>
# <auc:PackageOfMeasures>
# <auc:ReferenceCase IDref="Baseline"/>
# <auc:MeasureIDs>
# <auc:MeasureID IDref="Measure1"/>
# </auc:MeasureIDs>
# <auc:AnnualSavingsSiteEnergy>162654.89601696888</auc:AnnualSavingsSiteEnergy>
# </auc:PackageOfMeasures>
# </auc:ScenarioType>
# </auc:Scenario>
# KAF: for now, handle both Reports.Report and Report
scenarios = self._get_node(
'auc:BuildingSync.auc:Facilities.auc:Facility.auc:Reports.auc:Report.auc:Scenarios.auc:Scenario',
data, [])
if not scenarios:
scenarios = self._get_node(
'auc:BuildingSync.auc:Facilities.auc:Facility.auc:Report.auc:Scenarios.auc:Scenario',
data, [])
# check that this is a list; if not, make it a list or the loop won't work correctly
if isinstance(scenarios, dict):
# print("scenarios is a dict (only found 1...converting it to a list)")
scenarios_tmp = []
scenarios_tmp.append(scenarios)
scenarios = scenarios_tmp
for s in scenarios:
new_data = {
'id': s.get('@ID'),
'name': s.get('auc:ScenarioName'),
}
if s.get('auc:ScenarioType'):
node = s['auc:ScenarioType'].get('auc:PackageOfMeasures')
if node:
ref_case = self._get_node('auc:ReferenceCase', node, [])
if ref_case and ref_case.get('@IDref'):
new_data['reference_case'] = ref_case.get('@IDref')
# fixed naming of existing scenario fields
new_data['annual_site_energy_savings'] = node.get(
'auc:AnnualSavingsSiteEnergy')
new_data['annual_source_energy_savings'] = node.get(
'auc:AnnualSavingsSourceEnergy')
new_data['annual_cost_savings'] = node.get(
'auc:AnnualSavingsCost')
# new scenario fields
fuel_savings = node.get('auc:AnnualSavingsByFuels')
if fuel_savings:
fuel_nodes = fuel_savings.get(
'auc:AnnualSavingsByFuel')
if isinstance(fuel_nodes, dict):
fuel_savings_arr = []
fuel_savings_arr.append(fuel_nodes)
fuel_nodes = fuel_savings_arr
for f in fuel_nodes:
if f.get('auc:EnergyResource') == 'Electricity':
new_data['annual_electricity_savings'] = f.get(
'auc:AnnualSavingsNativeUnits')
# print("ELECTRICITY: {}".format(new_data['annual_electricity_savings']))
elif f.get('auc:EnergyResource') == 'Natural gas':
new_data['annual_natural_gas_savings'] = f.get(
'auc:AnnualSavingsNativeUnits')
# print("GAS: {}".format(new_data['annual_natural_gas_savings']))
all_resources = s.get('auc:AllResourceTotals')
if all_resources:
resource_nodes = all_resources.get(
'auc:AllResourceTotal')
# print("ANNUAL ENERGY: {}".format(resource_nodes))
# make it an array
if isinstance(resource_nodes, dict):
resource_nodes_arr = []
resource_nodes_arr.append(resource_nodes)
resource_nodes = resource_nodes_arr
for rn in resource_nodes:
if rn.get('auc:EndUse') == 'All end uses':
new_data['annual_site_energy'] = rn.get(
'auc:SiteEnergyUse')
new_data[
'annual_site_energy_use_intensity'] = rn.get(
'auc:SiteEnergyUseIntensity')
new_data['annual_source_energy'] = rn.get(
'auc:SourceEnergyUse')
new_data[
'annual_source_energy_use_intensity'] = rn.get(
'auc:SourceEnergyUseIntensity')
resources = []
resource_uses = s.get('auc:ResourceUses')
if resource_uses:
ru_nodes = resource_uses.get('auc:ResourceUse')
# print("ResourceUse: {}".format(ru_nodes))
if isinstance(ru_nodes, dict):
ru_nodes_arr = []
ru_nodes_arr.append(ru_nodes)
ru_nodes = ru_nodes_arr
for ru in ru_nodes:
# store resourceID and EnergyResource -- needed for TimeSeries
r = {}
r['id'] = ru.get('@ID')
r['type'] = ru.get('auc:EnergyResource')
r['units'] = ru.get('auc:ResourceUnits')
resources.append(r)
# just do these 2 types for now
if ru.get('auc:EnergyResource') == 'Electricity':
new_data['annual_electricity_energy'] = ru.get(
'auc:AnnualFuelUseConsistentUnits'
) # in MMBtu
# get demand as well
new_data['annual_peak_demand'] = ru.get(
'auc:AnnualPeakConsistentUnits') # in KW
elif ru.get('auc:EnergyResource') == 'Natural gas':
new_data['annual_natural_gas_energy'] = ru.get(
'auc:AnnualFuelUseConsistentUnits'
) # in MMBtu
# timeseries
timeseriesdata = s.get('auc:TimeSeriesData')
# need to know if CalculationMethod is modeled (for meters)
isVirtual = False
calcMethod = node.get('auc:CalculationMethod')
if calcMethod is not None:
isModeled = calcMethod.get('auc:Modeled')
if isModeled is not None:
isVirtual = True
if timeseriesdata:
timeseries = timeseriesdata.get('auc:TimeSeries')
if isinstance(timeseries, dict):
ts_nodes_arr = []
ts_nodes_arr.append(timeseries)
timeseries = ts_nodes_arr
new_data['meters'] = []
for ts in timeseries:
source_id = ts.get('auc:ResourceUseID').get(
'@IDref')
# print("SOURCE ID: {}".format(source_id))
source_unit = next((item for item in resources
if item['id'] == source_id),
None)
source_unit = source_unit[
'units'] if source_unit is not None else None
match = next((item for item in new_data['meters']
if item['source_id'] == source_id),
None)
if match is None:
# this source_id is not yet in meters, add it
meter = {}
meter['source_id'] = source_id
source = next((item for item in Meter.SOURCES
if item[1] == 'BuildingSync'),
None)
meter['source'] = source[0] # for BuildingSync
meter['is_virtual'] = isVirtual
typeMatch = next((item for item in resources
if item['id'] == source_id),
None)
typeMatch = typeMatch['type'].title(
) if typeMatch is not None else None
# print("TYPE MATCH: {}".format(type_match))
# For "Electricity", match on 'Electric - Grid'
tmp_type = "Electric - Grid" if typeMatch == 'Electricity' else typeMatch
theType = next((item
for item in Meter.ENERGY_TYPES
if item[1] == tmp_type), None)
# print("the type: {}".format(the_type))
theType = theType[
0] if theType is not None else None
meter['type'] = theType
meter['readings'] = []
new_data['meters'].append(meter)
# add reading connected to meter (use resourceID/source_id for matching)
reading = {}
# ignoring timezones...pretending all is in UTC for DB and Excel export
reading['start_time'] = pytz.utc.localize(
datetime.strptime(ts.get('auc:StartTimeStamp'),
"%Y-%m-%dT%H:%M:%S"))
reading['end_time'] = pytz.utc.localize(
datetime.strptime(ts.get('auc:EndTimeStamp'),
"%Y-%m-%dT%H:%M:%S"))
reading['reading'] = ts.get('auc:IntervalReading')
reading['source_id'] = source_id
reading['source_unit'] = source_unit
# append to appropriate meter (or don't import)
the_meter = next(
(item for item in new_data['meters']
if item['source_id'] == source_id), None)
if the_meter is not None:
the_meter['readings'].append(reading)
# print("METERS: {}".format(new_data['meters']))
# measures
new_data['measures'] = []
measures = self._get_node('auc:MeasureIDs.auc:MeasureID',
node, [])
if isinstance(measures, list):
for measure in measures:
if measure.get('@IDref', None):
new_data['measures'].append(
measure.get('@IDref'))
else:
if isinstance(measures, basestring):
# the measure is there, but it does not have an idref
continue
else:
if measures.get('@IDref', None):
new_data['measures'].append(
measures.get('@IDref'))
res['scenarios'].append(new_data)
# print("SCENARIOS: {}".format(res['scenarios']))
return res, messages
def snake_case(value):
# avoid circular dependency
from seed.models.measures import _snake_case
return _snake_case(value)
