def test_data_collection_json(): dc1 = DataCollection(item1="item1_value", item2="item2_value", tags=["tag1"]) dc1.add_data_collection(DataCollection(item1="item1_value", item2="item2_value", tags=["tag2"])) json_data = dc1.json() json.dumps(json_data) assert len(json_data["item1"]) == 2 assert len(json_data["item2"]) == 2 assert json_data["item1"][0]["value"] == "item1_value" assert json_data["item1"][0]["tags"] in [["tag1"], ["tag2"]]
def test_conditional_meter_without_params(): meter_yaml=""" !obj:eemeter.meter.Condition { condition: { "name": "electricity_present", } } """ meter = load(meter_yaml) data_collection = DataCollection(electricity_present=True) assert meter.evaluate(data_collection).count() == 0 data_collection = DataCollection(electricity_present=False) assert meter.evaluate(data_collection).count() == 0
def test_switch(): meter_yaml = """ !obj:eemeter.meter.Switch { target: {name: target}, cases: { 1: !obj:eemeter.meter.DummyMeter { input_mapping: { value: {name: value_one} }, output_mapping: { result: {} }, }, 2: !obj:eemeter.meter.DummyMeter { input_mapping: { value: {name: value_two} }, output_mapping: { result: {} }, }, 3: !obj:eemeter.meter.DummyMeter { input_mapping: { value: {name: value_three} }, output_mapping: { result: {} }, }, }, default: !obj:eemeter.meter.DummyMeter { input_mapping: { value: {name: value_default} }, output_mapping: { result: {} }, } } """ meter = load(meter_yaml) data_collection = DataCollection(target=1, value_one=1, value_two=2, value_three=3, value_default=4) result1 = meter.evaluate(data_collection) data_collection = DataCollection(target=2, value_one=1, value_two=2, value_three=3, value_default=4) result2 = meter.evaluate(data_collection) data_collection = DataCollection(target=3, value_one=1, value_two=2, value_three=3, value_default=4) result3 = meter.evaluate(data_collection) data_collection = DataCollection(target=4, value_one=1, value_two=2, value_three=3, value_default=4) result4 = meter.evaluate(data_collection) data_collection = DataCollection(value_one=1, value_two=2, value_three=3, value_default=4) result5 = meter.evaluate(data_collection) assert 1 == result1.get_data("result").value assert 2 == result2.get_data("result").value assert 3 == result3.get_data("result").value assert 4 == result4.get_data("result").value assert None == result5.get_data("result")
def test_sequential_meter(): meter_yaml = """ !obj:eemeter.meter.Sequence { sequence: [ !obj:eemeter.meter.DummyMeter { input_mapping: { "value": {}, }, output_mapping: { "result": {"name":"value1"}, }, }, !obj:eemeter.meter.DummyMeter { input_mapping: { "value": { "name": "value1"}, }, output_mapping: { "result": {}, }, }, !obj:eemeter.meter.DummyMeter { input_mapping: { "value": {}, }, output_mapping: { "result": {"name": "result1"}, }, }, ] } """ meter = load(meter_yaml) data_collection = DataCollection(value=10) result = meter.evaluate(data_collection) assert result.get_data("value") is None assert result.get_data("value1").value == 10 assert result.get_data("result").value == 10 assert result.get_data("result1").value == 10
def test_meets_thresholds(): meter_yaml = """ !obj:eemeter.meter.MeetsThresholds { input_mapping: { "one": {}, "two": {}, "three": {}, "four": {}, }, equations: [ ["one", "<", 1, 0, 0, "one_lt_zero"], ["two", "<=", 1, "two", 0, "two_lte_two"], ["three", ">", .5, "four", 0, "three_gt_half_four"], ["four", ">=", "two", 3, 0, "four_gte_twice_three"], ["four", ">=", 2, 1, "two", "four_gte_four"], ], output_mapping: { "one_lt_zero": {}, "two_lte_two": {}, "three_gt_half_four": {}, "four_gte_twice_three": {}, "four_gte_four": {}, }, } """ meter = load(meter_yaml) data_collection = DataCollection(one=1, two=2, three=3.0, four=4) result = meter.evaluate(data_collection) assert not result.get_data("one_lt_zero").value assert result.get_data("two_lte_two").value assert result.get_data("three_gt_half_four").value assert not result.get_data("four_gte_twice_three").value assert result.get_data("four_gte_four").value
def test_for(): meter_yaml = """ !obj:eemeter.meter.For { variable: { name: value, }, iterable: { name: iterable, }, meter: !obj:eemeter.meter.DummyMeter { input_mapping: { value: {} }, output_mapping: { result: {} }, } } """ meter = load(meter_yaml) iterable = [ { "value": 1, "tags": ["one"] }, { "value": 2, "tags": ["two"] }] data_collection = DataCollection(iterable=iterable) result = meter.evaluate(data_collection) assert 1 == result.get_data("result", tags=["one"]).value assert 2 == result.get_data("result", tags=["two"]).value
def evaluate(self, data_collection): """Determine the value of the target parameter and, according to that value, choose which meter to run. Returns ------- out : dict Contains the results of each meter run. """ output_data_collection = DataCollection() for i in data_collection.get_data(**self.iterable).value: variable_name = self.variable["name"] variable_value = i.get("value") variable_tags = self.variable.get("tags", []) data_container = DataContainer(name=variable_name, value=variable_value, tags=variable_tags) input_data_collection = data_collection.copy() input_data_collection.add_data(data_container) outputs = self.meter.evaluate(input_data_collection) tags = i.get("tags", []) outputs.add_tags(tags) output_data_collection.add_data_collection(outputs) return output_data_collection
def test_data_collection_add_data_collection(data_collection): dc = DataCollection() new_data = DataContainer("name", "new_data_value", ["tag1"]) dc.add_data(new_data) data_collection.add_data_collection(dc,tagspace=["tagspace"]) assert data_collection.get_data("name", tags=["tag1","tagspace"]).name == "name" with pytest.raises(TypeError): data_collection.add_data_collection("name", "tagspace")
def test_temperature_sensitivity_parameter_optimization( generated_consumption_data_1, gsod_722880_2012_2014_weather_source): meter_yaml = """ !obj:eemeter.meter.TemperatureSensitivityParameterOptimizationMeter { temperature_unit_str: "degF", model: !obj:eemeter.models.AverageDailyTemperatureSensitivityModel { cooling: True, heating: True, initial_params: { base_daily_consumption: 0, heating_slope: 0, cooling_slope: 0, heating_balance_temperature: 60, cooling_balance_temperature: 70, }, param_bounds: { base_daily_consumption: [0,2000], heating_slope: [0,200], cooling_slope: [0,200], heating_balance_temperature: [55,65], cooling_balance_temperature: [65,75], }, }, input_mapping: { "consumption_data": {}, "weather_source": {}, "energy_unit_str": {}, }, output_mapping: { "temp_sensitivity_params": {}, "n_days": {}, "average_daily_usages": {}, "estimated_average_daily_usages": {}, }, } """ meter = load(meter_yaml) cd, params = generated_consumption_data_1 data_collection = DataCollection( consumption_data=cd, weather_source=gsod_722880_2012_2014_weather_source, energy_unit_str="kWh") result = meter.evaluate(data_collection) assert_allclose(result.get_data('temp_sensitivity_params').value.to_list(), params.to_list(), rtol=RTOL, atol=ATOL) assert result.get_data('n_days') is not None assert result.get_data('average_daily_usages') is not None assert result.get_data('estimated_average_daily_usages') is not None
def evaluate(self, data_collection): """Determine the value of the target parameter and, according to that value, choose which meter to run. Returns ------- out : dict Contains the results of the meter which was run. """ item = data_collection.get_data(**self.target) if item is None: return DataCollection() meter = self.cases.get(item.value) if meter is None: if self.default is None: return DataCollection() else: meter = self.default output_data_collection = meter.evaluate(data_collection) return output_data_collection
def test_dummy_meter_auxiliary_inputs(): meter_yaml = """ !obj:eemeter.meter.DummyMeter { auxiliary_inputs: { "value": "aux" }, output_mapping: { "result": {}, }, } """ meter = load(meter_yaml) data_collection = DataCollection() result = meter.evaluate(data_collection) assert result.get_data(name="result").value == "aux"
def test_estimated_reading_consolidation_meter_single_fuel_type(consumption_data): meter_yaml = """ !obj:eemeter.meter.EstimatedReadingConsolidationMeter { input_mapping: {"consumption_data": {}}, output_mapping: {"consumption_data_no_estimated": {}}, } """ meter = load(meter_yaml) data_collection = DataCollection(consumption_data=consumption_data) result = meter.evaluate(data_collection) values = result.get_data("consumption_data_no_estimated").value.data.values assert_allclose(values, np.array([0, 0, 0, np.nan]), rtol=RTOL, atol=ATOL)
def test_conditional_meter(): meter_yaml=""" !obj:eemeter.meter.Condition { condition: { "name": "electricity_present" }, success: !obj:eemeter.meter.DummyMeter { input_mapping: {"value": {"name": "success"}}, output_mapping: {"result": {}}, }, failure: !obj:eemeter.meter.DummyMeter { input_mapping: {"value": {"name": "failure"}}, output_mapping: {"result": {}}, }, } """ meter = load(meter_yaml) data_collection = DataCollection(electricity_present=True, success="success", failure="failure") assert meter.evaluate(data_collection).get_data("result").value == "success" data_collection = DataCollection(electricity_present=False, success="success", failure="failure") assert meter.evaluate(data_collection).get_data("result").value == "failure"
def test_fuel_type_tag_filter(): meter_yaml = """ !obj:eemeter.meter.FuelTypeTagFilter { fuel_type_search_name: active_fuel_type, input_mapping: { active_fuel_type: {}, }, meter: !obj:eemeter.meter.Sequence { sequence: [ !obj:eemeter.meter.DummyMeter { input_mapping: { value: { name: active_fuel_type } }, output_mapping: { result: { name: result1 } }, }, !obj:eemeter.meter.DummyMeter { input_mapping: { value: {} }, output_mapping: { result: { name: result2 } }, } ] } } """ meter = load(meter_yaml) data_collection = DataCollection(active_fuel_type="electricity") data_collection_include = DataCollection(value="value_include") data_collection_exclude = DataCollection(value="value_exclude") data_collection.add_data_collection(data_collection_include, tagspace=["electricity"]) data_collection.add_data_collection(data_collection_exclude, tagspace=["natural_gas"]) output_data_collection = meter.evaluate(data_collection) assert output_data_collection.get_data("result1").value == "electricity" assert output_data_collection.get_data("result2").value == "value_include"
def test_tag_filter(): meter_yaml = """ !obj:eemeter.meter.TagFilter { meter: !obj:eemeter.meter.DummyMeter { input_mapping: { value: {} }, output_mapping: { result: {} }, } } """ meter = load(meter_yaml) data_collection = DataCollection() with pytest.raises(NotImplementedError): meter.evaluate(data_collection)
def evaluate(self, data_collection): """Overrides normal execution to evaluate meters in sequence, collecting outputs. Returns ------- out : dict Collected outputs from all meters in the sequence. """ input_data_collection = data_collection.copy() output_data_collection = DataCollection() for meter in self.sequence: meter_result = meter.evaluate(input_data_collection) input_data_collection.add_data_collection(meter_result) output_data_collection.add_data_collection(meter_result) return output_data_collection
def test_r_squared(): meter_yaml = """ !obj:eemeter.meter.RSquared { input_mapping: { "y": {}, "y_hat": {}}, output_mapping: { "r_squared": {}} } """ meter = load(meter_yaml) data_collection = DataCollection( y=np.array([12,13,414,12,23,12,32,np.nan]), y_hat=np.array([32,12,322,21,22,41,32,np.nan])) result = meter.evaluate(data_collection) assert_allclose(result.get_data("r_squared").value, 0.9276, rtol=RTOL, atol=ATOL)
def test_cvrmse(): meter_yaml = """ !obj:eemeter.meter.CVRMSE { input_mapping: { "y": {}, "y_hat": {}, "params": {}}, output_mapping: { "cvrmse": {}} } """ meter = load(meter_yaml) data_collection = DataCollection( y=np.array([12,13,414,12,23,12,32,np.nan]), y_hat=np.array([32,12,322,21,22,41,32,np.nan]), params=BaseloadHeatingModelParameterType([1,3,4])) result = meter.evaluate(data_collection) assert_allclose(result.get_data("cvrmse").value, 59.79, rtol=RTOL, atol=ATOL)
def evaluate(self, data_collection): """Overrides the evaluate method to evaluate either the `success` meter or the `failure` meter on the boolean value of a particular meter input stored in the element with the name `condition_parameter`. Returns ------- out : dict Collected outputs from either the success or the failure meter. """ if data_collection.get_data(**self.condition).value: meter = self.success else: meter = self.failure if meter is None: return DataCollection() else: return meter.evaluate(data_collection)
def test_tutorial(): project = get_example_project("94087") meter = DefaultResidentialMeter() results = meter.evaluate(DataCollection(project=project)) electricity_usage_pre = results.get_data("annualized_usage", ["electricity", "baseline"]).value electricity_usage_post = results.get_data( "annualized_usage", ["electricity", "reporting"]).value natural_gas_usage_pre = results.get_data("annualized_usage", ["natural_gas", "baseline"]).value natural_gas_usage_post = results.get_data( "annualized_usage", ["natural_gas", "reporting"]).value electricity_savings = (electricity_usage_pre - electricity_usage_post) / electricity_usage_pre natural_gas_savings = (natural_gas_usage_pre - natural_gas_usage_post) / natural_gas_usage_pre json_data = results.json() assert "consumption" in json_data assert "weather_source" in json_data assert "gross_savings" in json_data
def data_collection(data_container_name_value): dc = DataCollection() dc.add_data(data_container_name_value) return dc
def test_data_collection_creation_shortcut(): dc = DataCollection(item1="item1_value", item2="item2_value", tags=["tag"]) assert dc.get_data("item1",tags=["tag"]).name == "item1" assert dc.get_data("item1",tags=["tag"]).value == "item1_value" assert dc.get_data("item2",tags=["tag"]).name == "item2" assert dc.get_data("item2",tags=["tag"]).value == "item2_value"
def test_default_residential_meter(default_residential_outputs_1, gsod_722880_2012_2014_weather_source, tmy3_722880_weather_source): elec_consumption_data, gas_consumption_data, \ elec_params, gas_params, \ elec_annualized_usage, gas_annualized_usage, \ elec_gross_savings, gas_gross_savings, \ elec_rmse, gas_rmse, \ elec_r_squared, gas_r_squared, \ elec_consumption_kWh_per_day, gas_consumption_kWh_per_day, \ elec_consumption_n_days, gas_consumption_n_days, \ temp_unit, retrofit_start_date, retrofit_completion_date, \ cdd_tmy, hdd_tmy, total_cdd, total_hdd = \ default_residential_outputs_1 meter = DefaultResidentialMeter(temperature_unit_str=temp_unit) location = Location(station="722880") baseline_period = Period(datetime(2012,1,1,tzinfo=pytz.utc),retrofit_start_date) reporting_period = Period(retrofit_completion_date, datetime(2014,12,31,tzinfo=pytz.utc)) project = Project(location, [elec_consumption_data, gas_consumption_data], baseline_period, reporting_period) data_collection = DataCollection(project=project) result = meter.evaluate(data_collection) assert_allclose(result.get_data('annualized_usage', tags=['electricity','baseline']).value, elec_annualized_usage, rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('annualized_usage', tags=['natural_gas','baseline']).value, gas_annualized_usage, rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('annualized_usage', tags=['electricity','reporting']).value, elec_annualized_usage, rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('annualized_usage', tags=['natural_gas','reporting']).value, gas_annualized_usage, rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('gross_savings', tags=['electricity']).value, elec_gross_savings, rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('gross_savings', tags=['natural_gas']).value, gas_gross_savings, rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('average_daily_usages', tags=['electricity', 'baseline']).value, elec_consumption_kWh_per_day[:17], rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('average_daily_usages', tags=['natural_gas', 'baseline']).value, gas_consumption_kWh_per_day[:17], rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('average_daily_usages', tags=['electricity', 'reporting']).value, elec_consumption_kWh_per_day[20:], rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('average_daily_usages', tags=['natural_gas', 'reporting']).value, gas_consumption_kWh_per_day[20:], rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('cdd_tmy', tags=['electricity', 'baseline']).value, cdd_tmy, rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('cdd_tmy', tags=['natural_gas', 'baseline']).value, cdd_tmy, rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('cdd_tmy', tags=['electricity', 'reporting']).value, cdd_tmy, rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('cdd_tmy', tags=['natural_gas', 'reporting']).value, cdd_tmy, rtol=RTOL, atol=ATOL) elec_rmse_results = result.search('rmse', ['electricity']) assert elec_rmse_results.count() == 2 for data_container in elec_rmse_results.iteritems(): assert_allclose(data_container.value, elec_rmse, rtol=RTOL, atol=ATOL) gas_rmse_results = result.search('rmse', ['natural_gas']) assert gas_rmse_results.count() == 2 for data_container in gas_rmse_results.iteritems(): assert_allclose(data_container.value, gas_rmse, rtol=RTOL, atol=ATOL) elec_r_squared_results = result.search('r_squared', ['electricity']) assert elec_r_squared_results.count() == 2 for data_container in elec_r_squared_results.iteritems(): assert_allclose(data_container.value, elec_r_squared, rtol=RTOL, atol=ATOL) gas_r_squared_results = result.search('r_squared', ['natural_gas']) assert gas_r_squared_results.count() == 2 for data_container in gas_r_squared_results.iteritems(): assert_allclose(data_container.value, gas_r_squared, rtol=RTOL, atol=ATOL)
def test_annualized_usage_meter( generated_consumption_data_with_annualized_usage_1, gsod_722880_2012_2014_weather_source, tmy3_722880_weather_source): meter_yaml = """ !obj:eemeter.meter.Sequence { sequence: [ !obj:eemeter.meter.TemperatureSensitivityParameterOptimizationMeter { temperature_unit_str: "degF", model: !obj:eemeter.models.AverageDailyTemperatureSensitivityModel &model { cooling: True, heating: True, initial_params: { base_daily_consumption: 0, heating_slope: 0, cooling_slope: 0, heating_balance_temperature: 60, cooling_balance_temperature: 70, }, param_bounds: { base_daily_consumption: [0,2000], heating_slope: [0,200], cooling_slope: [0,200], heating_balance_temperature: [55,65], cooling_balance_temperature: [65,75], }, }, input_mapping: { consumption_data: {}, weather_source: {}, energy_unit_str: {}, }, output_mapping: { temp_sensitivity_params: {name: model_params}, }, }, !obj:eemeter.meter.AnnualizedUsageMeter { temperature_unit_str: "degF", model: *model, input_mapping: { model_params: {}, weather_normal_source: {}, }, output_mapping: { annualized_usage: {}, }, } ] } """ meter = load(meter_yaml) cd, params, annualized_usage = \ generated_consumption_data_with_annualized_usage_1 data_collection = DataCollection( consumption_data=cd, weather_source=gsod_722880_2012_2014_weather_source, weather_normal_source=tmy3_722880_weather_source, energy_unit_str="kWh") result = meter.evaluate(data_collection) assert_allclose(result.get_data('model_params').value.to_list(), params.to_list(), rtol=RTOL, atol=ATOL) assert_allclose(result.get_data('annualized_usage').value, annualized_usage, rtol=RTOL, atol=ATOL)
def test_data_collection_init_tag_type_errors(): with pytest.raises(TypeError): DataCollection(tags="tag")
def read_meter_data(trace_filename, project_info_filename, project_id=None, weather=True, merge_series=True): """Read meter data from a raw XML file source, obtain matching project information from a separate CSV file. Fetches the corresponding weather data, when requested, too. Parameters ========== trace_filename: str Filename of XML meter trace. project_info_filename: str Filename of CSV file containing project info. project_id: str Manually provide the project ID used in `project_info_filename`. If `None`, the first part of `trace_filename` before a `_` is used. weather: bool `True` will obtain weather (temperature) data. merge_series: bool `True` will return a `pandas.DataFrame` with merged consumption and temperature data. Returns ======= A `DataCollection` object with the following fields: project_info: `pandas.DataFrame` Contains columns for project properties. baseline_end: `pandas.Datetime` End date of the baseline period. consumption_data: `eemeter.consumption.ConsumptionData` Consumption data object. consumption_data_freq: `pandas.DataFrame` Consumption data with normalized frequency. If :samp:`weather=True`: weather_source: `eemeter.ISDWeatherSource` Weather source object. weather_data: `pandas.DataFrame` Temperature observations in, degF, with frequency matching `consumption_data_freq`. Values are averaged if raw temperature observations are lower frequency. If :samp:`merge_series=True`: cons_weather_data: `pandas.DataFrame` Merged consumption and temperature data. """ from eemeter.meter import DataCollection, DataContainer from eemeter.parsers import ESPIUsageParser # TODO: New API. #from eemeter.structures import ( # EnergyTrace, # EnergyTraceSet, # Intervention, # ZIPCodeSite, # Project #) #from eemeter.io.parsers import ESPIUsageParser from eemeter.weather import ISDWeatherSource import pandas as pd import os with open(trace_filename, 'r') as f: parser = ESPIUsageParser(f.read()) consumption_datas = list(parser.get_consumption_data_objects()) cons_data_obj = consumption_datas[0] all_projects_info = pd.read_csv(project_info_filename) fuel_type_map = {'electricity': 'E', 'natural_gas': 'NG'} if project_id is None: project_id = os.path.basename(trace_filename).split("_")[0] fuel_type = fuel_type_map[cons_data_obj.fuel_type] project_info = all_projects_info.query('project_id == "{}" and\ fuel_type == "{}"'.format( project_id, fuel_type)) baseline_end = pd.to_datetime(project_info.baseline_period_end.tolist()[0], utc=True) # Sometimes the data have differing observation frequencies, # so choose the most common one (in the usage data) and align # everything to that. cons_index_diff = cons_data_obj.data.index.to_series().diff(periods=1) new_freq = pd.value_counts(cons_index_diff).argmax() cons_data = cons_data_obj.data.tz_convert("UTC") cons_data = cons_data.resample(new_freq).mean() res = DataCollection(project_info=project_info, baseline_end=baseline_end, consumption_data=cons_data_obj, consumption_data_freq=cons_data) if weather: station = unicode(project_info.weather_station.tolist()[0]) ws = ISDWeatherSource(station) res.add_data(DataContainer("weather_source", ws, None)) ws.add_year_range(cons_data.index.min().year, cons_data.index.max().year) weather_data = ws._unit_convert(ws.tempC, "degF").tz_localize("UTC") weather_data = weather_data.resample(new_freq).mean() res.add_data(DataContainer("weather_data", weather_data, None)) if weather_data.empty: raise ValueError("No weather data") if merge_series: cons_weather_data = pd.concat([cons_data, weather_data], axis=1, join="inner") cons_weather_data.columns = ['usage', 'temp'] res.add_data( DataContainer("cons_weather_data", cons_weather_data, None)) return res