def setup_class(cls):
# initialize with data to avoid pulling multiple times
cls.sim_config = Config.make_sim_config(
identifier=[
"DYD_dummy_data",
], # test file
latitude=33.481136,
longitude=-112.078232,
start_utc=[
"2018-01-01 00:00:00",
],
end_utc=[
"2018-12-31 23:55:00",
],
min_sim_period="3D",
min_chunk_period="30D",
sim_step_size_seconds=300,
output_step_size_seconds=300,
)
cls.data_client = DataClient(
source=LocalSource(
local_cache=os.environ.get("LOCAL_CACHE_DIR"),
data_spec=DonateYourDataSpec(),
),
destination=LocalDestination(
local_cache=os.environ.get("LOCAL_CACHE_DIR"),
data_spec=DonateYourDataSpec(),
),
)
cls.data_client.sim_config = cls.sim_config.iloc[0]
cls.data_client.get_data()
def setup_class(cls):
# initialize with data to avoid pulling multiple times
cls.sim_config = Config.make_sim_config(
identifier=[
os.environ.get("TEST_FLATFILES_IDENTIFIER_MISSING"), # missing
os.environ.get("TEST_FLATFILES_IDENTIFIER_FULL"), # full
"9999999", # file not found
],
latitude=33.481136,
longitude=-112.078232,
start_utc="2018-01-01 00:00:00",
end_utc="2018-12-31 00:00:00",
min_sim_period="7D",
min_chunk_period="30D",
sim_step_size_seconds=300,
output_step_size_seconds=300,
)
cls.data_clients = []
# set local_cache=None to test connection with GCS
cls.data_client = DataClient(
source=GCSFlatFilesSource(
gcp_project=os.environ.get("FLATFILE_GOOGLE_CLOUD_PROJECT"),
gcs_uri_base=os.environ.get("FLATFILES_GCS_URI_BASE"),
local_cache=None,
),
destination=LocalDestination(
local_cache=os.environ.get("LOCAL_CACHE_DIR"),
data_spec=FlatFilesSpec(),
),
nrel_dev_api_key=os.environ.get("NREL_DEV_API_KEY"),
nrel_dev_email=os.environ.get("NREL_DEV_EMAIL"),
archive_tmy3_dir=os.environ.get("ARCHIVE_TMY3_DIR"),
archive_tmy3_meta=os.environ.get("ARCHIVE_TMY3_META"),
archive_tmy3_data_dir=os.environ.get("ARCHIVE_TMY3_DATA_DIR"),
ep_tmy3_cache_dir=os.environ.get("EP_TMY3_CACHE_DIR"),
simulation_epw_dir=os.environ.get("SIMULATION_EPW_DIR"),
)
for _idx, _sim_config in cls.sim_config.iterrows():
dc = copy.deepcopy(cls.data_client)
dc.sim_config = _sim_config
if _sim_config["identifier"] in [
"9999999",
]:
with pytest.raises(ValueError):
dc.get_data()
else:
dc.get_data()
cls.data_clients.append(dc)
def test_sim_config(self):
return (Config.make_sim_config(
identifier=[
"DYD_dummy_data",
], # has full data periods
latitude=41.8781,
longitude=-87.6298,
start_utc="2018-01-01",
end_utc="2018-01-04",
min_sim_period="1D",
sim_step_size_seconds=300,
output_step_size_seconds=300,
).iloc[0].to_dict())
def setup_class(cls):
# initialize with data to avoid pulling multiple times
cls.sim_config = Config.make_sim_config(
identifier=[
"d310f1c1f600c374d8975c753f7c0fb8de9c96b1",
"8c00c9bb17bfcca53809cb1b2d033a448bc017df", # has full data periods
"6773291da5427ae87d34bb75022ee54ee3b1fc17", # file not found
],
latitude=33.481136,
longitude=-112.078232,
start_utc=[
"2018-01-01 00:00:00",
"2018-01-01 00:00:00",
"2018-01-01 00:00:00",
],
end_utc=[
"2018-12-31 23:55:00",
"2018-12-31 23:55:00",
"2018-12-31 23:55:00",
],
min_sim_period="7D",
min_chunk_period="30D",
sim_step_size_seconds=300,
output_step_size_seconds=300,
)
cls.data_clients = []
# set local_cache=None to avoid caching locally and always testing connection with GCS
cls.data_client = DataClient(
source=GCSDYDSource(
gcp_project=os.environ.get("DYD_GOOGLE_CLOUD_PROJECT"),
gcs_uri_base=os.environ.get("DYD_GCS_URI_BASE"),
local_cache=None,
),
destination=LocalDestination(data_spec=DonateYourDataSpec(), ),
)
for _idx, _sim_config in cls.sim_config.iterrows():
dc = copy.deepcopy(cls.data_client)
dc.sim_config = _sim_config
if _sim_config["identifier"] in [
"6773291da5427ae87d34bb75022ee54ee3b1fc17",
]:
with pytest.raises(ValueError):
dc.get_data()
else:
dc.get_data()
cls.data_clients.append(dc)
def setup_class(cls):
# initialize with data to avoid pulling multiple times
cls.sim_config = Config.make_sim_config(
identifier=[
"DYD_dummy_data",
], # test file
latitude=33.481136,
longitude=-112.078232,
start_utc=[
"2018-01-01 00:00:00",
],
end_utc=[
"2018-12-31 23:55:00",
],
min_sim_period="3D",
min_chunk_period="30D",
sim_step_size_seconds=300,
output_step_size_seconds=300,
)
cls.data_clients = []
cls.data_client = DataClient(
source=LocalSource(
local_cache=os.environ.get("LOCAL_CACHE_DIR"),
data_spec=DonateYourDataSpec(),
),
destination=LocalDestination(
local_cache=os.environ.get("LOCAL_CACHE_DIR"),
data_spec=DonateYourDataSpec(),
),
nrel_dev_api_key=os.environ.get("NREL_DEV_API_KEY"),
nrel_dev_email=os.environ.get("NREL_DEV_EMAIL"),
archive_tmy3_dir=os.environ.get("ARCHIVE_TMY3_DIR"),
archive_tmy3_meta=os.environ.get("ARCHIVE_TMY3_META"),
archive_tmy3_data_dir=os.environ.get("ARCHIVE_TMY3_DATA_DIR"),
ep_tmy3_cache_dir=os.environ.get("EP_TMY3_CACHE_DIR"),
simulation_epw_dir=os.environ.get("SIMULATION_EPW_DIR"),
)
for _idx, _sim_config in cls.sim_config.iterrows():
dc = copy.deepcopy(cls.data_client)
dc.sim_config = _sim_config
dc.get_data()
cls.data_clients.append(dc)
def setup_class(cls):
# basic IDF file found in all EnergyPlus installations
cls.dummy_idf_name = "Furnace.idf"
cls.dummy_weather_name = "USA_IL_Chicago-OHare.Intl.AP.725300_TMY3.epw"
# make test/ dirs
EnergyPlusBuildingModel.make_directories()
cls.dummy_idf_path = os.path.join(os.environ.get("IDF_DIR"),
cls.dummy_idf_name)
cls.dummy_weather_file = os.path.join(os.environ.get("WEATHER_DIR"),
cls.dummy_weather_name)
# if dummy files don't exist copy them from E+ installations
if not os.path.isfile(cls.dummy_idf_path):
_fpath = os.path.join(os.environ.get("EPLUS_DIR"), "ExampleFiles",
cls.dummy_idf_name)
shutil.copyfile(_fpath, cls.dummy_idf_path)
if not os.path.isfile(cls.dummy_weather_file):
_fpath = os.path.join(
os.environ.get("EPLUS_DIR"),
"WeatherData",
cls.dummy_weather_name,
)
shutil.copyfile(_fpath, cls.dummy_weather_file)
cls.idf = IDFPreprocessor(idf_file=cls.dummy_idf_path,
timesteps_per_hour=12)
cls.step_size = int(3600.0 / cls.idf.timesteps_per_hour)
cls.test_sim_config = (
Config.make_sim_config(
identifier=[
"2df6959cdf502c23f04f3155758d7b678af0c631", # has full data periods
],
latitude=33.481136,
longitude=-112.078232,
start_utc="2018-05-16",
end_utc="2018-05-26",
min_sim_period="1D",
min_chunk_period="30D",
sim_step_size_seconds=60,
output_step_size_seconds=300,
).iloc[0].to_dict())
def setup_class(cls):
# initialize with data to avoid pulling multiple times
cls.sim_config = Config.make_sim_config(
identifier=["9958f46d13419344ec0c21fb60f9b0b3990ac0ef"],
latitude=33.481136,
longitude=-112.078232,
start_utc=[
"2018-01-01 00:00:00",
],
end_utc=[
"2018-12-31 23:55:00",
],
min_sim_period="7D",
min_chunk_period="30D",
sim_step_size_seconds=300,
output_step_size_seconds=300,
)
cls.data_clients = []
cls.data_client = DataClient(
source=GCSDYDSource(
gcp_project=os.environ.get("DYD_GOOGLE_CLOUD_PROJECT"),
gcs_uri_base=os.environ.get("DYD_GCS_URI_BASE"),
),
destination=LocalDestination(
local_cache=os.environ.get("LOCAL_CACHE_DIR"),
data_spec=DonateYourDataSpec(),
),
nrel_dev_api_key=os.environ.get("NREL_DEV_API_KEY"),
nrel_dev_email=os.environ.get("NREL_DEV_EMAIL"),
archive_tmy3_dir=os.environ.get("ARCHIVE_TMY3_DIR"),
archive_tmy3_meta=os.environ.get("ARCHIVE_TMY3_META"),
archive_tmy3_data_dir=os.environ.get("ARCHIVE_TMY3_DATA_DIR"),
ep_tmy3_cache_dir=os.environ.get("EP_TMY3_CACHE_DIR"),
simulation_epw_dir=os.environ.get("SIMULATION_EPW_DIR"),
)
for _idx, _sim_config in cls.sim_config.iterrows():
dc = copy.deepcopy(cls.data_client)
dc.sim_config = _sim_config
dc.get_data()
cls.data_clients.append(dc)
def setup_class(cls):
# initialize with data to avoid pulling multiple times
cls.sim_config = Config.make_sim_config(
identifier=[
"DYD_dummy_data", # test file
],
latitude=33.481136,
longitude=-112.078232,
start_utc=[
"2018-01-01 00:00:00",
],
end_utc=[
"2018-12-31 23:55:00",
],
min_sim_period="3D",
min_chunk_period="30D",
sim_step_size_seconds=300,
output_step_size_seconds=300,
)
cls.data_client = DataClient(
source=LocalSource(
local_cache=os.environ.get("LOCAL_CACHE_DIR"),
data_spec=DonateYourDataSpec(),
),
destination=GCSDestination(
gcp_project=os.environ.get("BCS_GOOGLE_CLOUD_PROJECT"),
gcs_uri_base=os.environ.get("BCS_OUTPUT_GCS_URI_BASE"),
data_spec=DonateYourDataSpec(),
local_cache=None,
),
nrel_dev_api_key=os.environ.get("NREL_DEV_API_KEY"),
nrel_dev_email=os.environ.get("NREL_DEV_EMAIL"),
archive_tmy3_dir=os.environ.get("ARCHIVE_TMY3_DIR"),
archive_tmy3_meta=os.environ.get("ARCHIVE_TMY3_META"),
archive_tmy3_data_dir=os.environ.get("ARCHIVE_TMY3_DATA_DIR"),
ep_tmy3_cache_dir=os.environ.get("EP_TMY3_CACHE_DIR"),
simulation_epw_dir=os.environ.get("SIMULATION_EPW_DIR"),
)
cls.data_client.sim_config = cls.sim_config.iloc[0]
cls.data_client.get_data()
def test_generate_dummy_data(self):
_sim_config = Config.make_sim_config(
identifier=[
"generated_dummy_data",
], # test file
latitude=33.481136,
longitude=-112.078232,
start_utc=[
"2018-01-01 00:00:00",
],
end_utc=[
"2018-12-31 23:55:00",
],
min_sim_period="3D",
min_chunk_period="30D",
sim_step_size_seconds=300,
output_step_size_seconds=300,
)
_df = DataClient.generate_dummy_data(sim_config=_sim_config,
spec=DonateYourDataSpec())
assert len(_df) == 105120
assert all(_df["Schedule"].value_counts().values == [74460, 30660])
def test_simulator(self, test_params):
_config_params = test_params["config"]
test_sim_config = Config.make_sim_config(
identifier=_config_params["identifier"],
latitude=_config_params["latitude"],
longitude=_config_params["longitude"],
start_utc=_config_params["start_utc"],
end_utc=_config_params["end_utc"],
min_sim_period=_config_params["min_sim_period"],
sim_step_size_seconds=_config_params["sim_step_size_seconds"],
output_step_size_seconds=_config_params[
"output_step_size_seconds"
],
)
dc = DataClient(
source=self.get_data_source(test_params["data_client"]),
destination=self.get_data_destination(test_params["data_client"]),
nrel_dev_api_key=os.environ.get("NREL_DEV_API_KEY"),
nrel_dev_email=os.environ.get("NREL_DEV_EMAIL"),
archive_tmy3_meta=os.environ.get("ARCHIVE_TMY3_META"),
archive_tmy3_data_dir=os.environ.get("ARCHIVE_TMY3_DATA_DIR"),
ep_tmy3_cache_dir=os.environ.get("EP_TMY3_CACHE_DIR"),
simulation_epw_dir=os.environ.get("SIMULATION_EPW_DIR"),
)
# test HVAC data returns dict of non-empty pd.DataFrame
master = Simulator(
data_client=dc,
sim_config=test_sim_config,
building_models=[
self.get_building_model(test_params["building_model"])
],
controller_models=[
self.get_controller_model(test_params["controller_model"])
],
state_estimator_models=[
self.get_state_estimator_model(
test_params["state_estimator_model"]
)
],
)
master.simulate(local=True, preprocess_check=False)
# read back stored output and check it
sim_name = master.simulations[0].sim_name
_fpath = os.path.join(
master.simulations[0].data_client.destination.local_cache,
master.simulations[0].data_client.destination.operator_name,
sim_name
+ "."
+ master.simulations[0].data_client.destination.file_extension,
)
r_df = pd.read_parquet(_fpath)
t_ctrl_name = [
_k
for _k, _v in master.simulations[
0
].data_client.destination.data_spec.full.spec.items()
if _v["internal_state"] == STATES.TEMPERATURE_CTRL
][0]
humidity_name = [
_k
for _k, _v in master.simulations[
0
].data_client.destination.data_spec.full.spec.items()
if _v["internal_state"] == STATES.THERMOSTAT_HUMIDITY
][0]
mean_thermostat_temperature = (
master.simulations[0].output[STATES.THERMOSTAT_TEMPERATURE].mean()
)
mean_thermostat_humidity = (
master.simulations[0].output[STATES.THERMOSTAT_HUMIDITY].mean()
)
output_format_mean_thermostat_temperature = r_df[t_ctrl_name].mean()
output_format_mean_thermostat_humidity = r_df[humidity_name].mean()
assert (
pytest.approx(
test_params["expected_result"]["mean_thermostat_temperature"]
)
== mean_thermostat_temperature
)
assert (
pytest.approx(
test_params["expected_result"]["mean_thermostat_humidity"]
)
== mean_thermostat_humidity
)
assert (
pytest.approx(
test_params["expected_result"][
"output_format_mean_thermostat_temperature"
]
)
== output_format_mean_thermostat_temperature
)
assert (
pytest.approx(
test_params["expected_result"][
"output_format_mean_thermostat_humidity"
]
)
== output_format_mean_thermostat_humidity
)
def setup_class(cls):
# initialize with data to avoid pulling multiple times
cls.sim_config = Config.make_sim_config(
identifier=[
"d310f1c1f600c374d8975c753f7c0fb8de9c96b1",
"8c00c9bb17bfcca53809cb1b2d033a448bc017df", # has full data periods
"6773291da5427ae87d34bb75022ee54ee3b1fc17", # file not found
],
latitude=33.481136,
longitude=-112.078232,
start_utc=[
"2018-01-01 00:00:00",
"2018-01-01 00:00:00",
"2018-01-01 00:00:00",
],
end_utc=[
"2018-12-31 23:55:00",
"2018-12-31 23:55:00",
"2018-12-31 23:55:00",
],
min_sim_period="7D",
min_chunk_period="30D",
sim_step_size_seconds=300,
output_step_size_seconds=300,
)
cls.data_clients = []
# set local_cache=None to test connection with GCS
cls.data_client = DataClient(
source=GCSDYDSource(
gcp_project=os.environ.get("DYD_GOOGLE_CLOUD_PROJECT"),
gcs_uri_base=os.environ.get("DYD_GCS_URI_BASE"),
local_cache=None,
),
destination=LocalDestination(
local_cache=os.environ.get("LOCAL_CACHE_DIR"),
data_spec=DonateYourDataSpec(),
),
nrel_dev_api_key=os.environ.get("NREL_DEV_API_KEY"),
nrel_dev_email=os.environ.get("NREL_DEV_EMAIL"),
archive_tmy3_dir=os.environ.get("ARCHIVE_TMY3_DIR"),
archive_tmy3_meta=os.environ.get("ARCHIVE_TMY3_META"),
archive_tmy3_data_dir=os.environ.get("ARCHIVE_TMY3_DATA_DIR"),
ep_tmy3_cache_dir=os.environ.get("EP_TMY3_CACHE_DIR"),
simulation_epw_dir=os.environ.get("SIMULATION_EPW_DIR"),
)
for _idx, _sim_config in cls.sim_config.iterrows():
dc = copy.deepcopy(cls.data_client)
dc.sim_config = _sim_config
if _sim_config["identifier"] in [
"6773291da5427ae87d34bb75022ee54ee3b1fc17",
]:
with pytest.raises(ValueError):
dc.get_data()
else:
dc.get_data()
cls.data_clients.append(dc)
def test_simulator(self, test_params):
_config_params = test_params["config"]
test_sim_config = Config.make_sim_config(
identifier=_config_params["identifier"],
latitude=_config_params["latitude"],
longitude=_config_params["longitude"],
start_utc=_config_params["start_utc"],
end_utc=_config_params["end_utc"],
min_sim_period=_config_params["min_sim_period"],
sim_step_size_seconds=_config_params["sim_step_size_seconds"],
output_step_size_seconds=_config_params["output_step_size_seconds"],
)
_epw_path = None
if test_params["data_client"].get("epw_name"):
_epw_path = self.get_epw_path(test_params["data_client"].get("epw_name"))
# do not use NREL data in test cases in case it changes or becomes unavailable
dc = DataClient(
source=self.get_data_source(test_params["data_client"]),
destination=self.get_data_destination(test_params["data_client"]),
nrel_dev_api_key=None,
epw_path=_epw_path,
)
# test HVAC data returns dict of non-empty pd.DataFrame
master = Simulator(
data_client=dc,
sim_config=test_sim_config,
building_models=[self.get_building_model(test_params["building_model"])],
controller_models=[
self.get_controller_model(test_params["controller_model"])
],
state_estimator_models=[
self.get_state_estimator_model(test_params["state_estimator_model"])
],
)
logger.info("calling master.simulate ... ")
master.simulate(local=True, preprocess_check=False)
logger.info("done master.simulate")
# read back stored output and check it
sim_name = master.simulations[0].sim_name
_fpath = os.path.join(
master.simulations[0].data_client.destination.local_cache,
master.simulations[0].data_client.destination.operator_name,
sim_name
+ "."
+ master.simulations[0].data_client.destination.file_extension,
)
r_df = pd.read_parquet(_fpath)
t_ctrl_name = [
_k
for _k, _v in master.simulations[
0
].data_client.destination.data_spec.full.spec.items()
if _v["internal_state"] == STATES.TEMPERATURE_CTRL
][0]
humidity_name = [
_k
for _k, _v in master.simulations[
0
].data_client.destination.data_spec.full.spec.items()
if _v["internal_state"] == STATES.THERMOSTAT_HUMIDITY
][0]
mean_thermostat_temperature = (
master.simulations[0].output[STATES.THERMOSTAT_TEMPERATURE].mean()
)
mean_thermostat_humidity = (
master.simulations[0].output[STATES.THERMOSTAT_HUMIDITY].mean()
)
output_format_mean_thermostat_temperature = r_df[t_ctrl_name].mean()
output_format_mean_thermostat_humidity = r_df[humidity_name].mean()
# print out values in case of slight divergence to avoid re-running tests
logger.info(
f"\nmean_thermostat_temperature= {mean_thermostat_temperature}\n"
+ f"mean_thermostat_humidity= {mean_thermostat_humidity}\n"
+ f"output_format_mean_thermostat_temperature= {output_format_mean_thermostat_temperature}\n"
+ f"output_format_mean_thermostat_humidity= {output_format_mean_thermostat_humidity}\n"
)
assert (
pytest.approx(test_params["expected_result"]["mean_thermostat_temperature"])
== mean_thermostat_temperature
)
assert (
pytest.approx(test_params["expected_result"]["mean_thermostat_humidity"])
== mean_thermostat_humidity
)
assert (
pytest.approx(
test_params["expected_result"][
"output_format_mean_thermostat_temperature"
]
)
== output_format_mean_thermostat_temperature
)
assert (
pytest.approx(
test_params["expected_result"]["output_format_mean_thermostat_humidity"]
)
== output_format_mean_thermostat_humidity
)
def test_make_epw_file(self):
_start_utc = "2019-01-17"
_end_utc = "2019-01-19"
_step_size_seconds = 300
_sim_step_size_seconds = 60
_data = pd.DataFrame({
STATES.DATE_TIME:
pd.date_range(
start=_start_utc,
end=_end_utc,
freq=f"{_sim_step_size_seconds}S",
tz="utc",
)
})
sim_config = Config.make_sim_config(
identifier="511863952006",
latitude=43.798577,
longitude=-79.239087,
start_utc=_start_utc,
end_utc=_end_utc,
min_sim_period="1D",
sim_step_size_seconds=_sim_step_size_seconds,
output_step_size_seconds=_step_size_seconds,
).iloc[0]
_internal_timezone = DateTimeChannel.get_timezone(
sim_config["latitude"], sim_config["longitude"])
internal_spec = Internal()
datetime_channel = DateTimeChannel(
data=_data[internal_spec.intersect_columns(
_data.columns, internal_spec.datetime.spec)],
spec=internal_spec.datetime,
latitude=sim_config["latitude"],
longitude=sim_config["longitude"],
internal_timezone=_internal_timezone,
)
weather_channel = WeatherChannel(
data=pd.DataFrame(),
spec=internal_spec,
nrel_dev_api_key=os.environ.get("NREL_DEV_API_KEY"),
nrel_dev_email=os.environ.get("NREL_DEV_EMAIL"),
archive_tmy3_dir=os.environ.get("ARCHIVE_TMY3_DIR"),
archive_tmy3_meta=os.environ.get("ARCHIVE_TMY3_META"),
archive_tmy3_data_dir=os.environ.get("ARCHIVE_TMY3_DATA_DIR"),
ep_tmy3_cache_dir=os.environ.get("EP_TMY3_CACHE_DIR"),
nsrdb_cache_dir=os.environ.get("NSRDB_CACHE_DIR"),
simulation_epw_dir=os.environ.get("SIMULATION_EPW_DIR"),
)
weather_channel.get_epw_data(sim_config, datetime_channel)
epw_path = weather_channel.make_epw_file(
sim_config=sim_config,
datetime_channel=datetime_channel,
epw_step_size_seconds=_step_size_seconds,
)
assert weather_channel.data.empty == False
assert (pytest.approx(weather_channel.data[
STATES.OUTDOOR_TEMPERATURE].mean()) == 1.78746962860115)