def test_cache_key_valid_types():
key1 = cache_key(["foo", "bar"], 4, "other")
assert (("foo", "bar"), 4, "other") == key1
key2 = cache_key(True)
assert (True, ) == key2
key3 = cache_key({1, 2, 2, 3})
assert ((1, 2, 3), ) == key3
key4 = cache_key(None)
assert ("null", ) == key4
def _get_data_internal(self, filepath):
key = cache_key(filepath)
cached = _cache.get(key)
if cached is not None:
return cached
with self.data_access.get(filepath) as (f, path):
data = self._read(f, path)
_cache.put(key, data)
return data
def test_mem_cache_put_version_never_changes():
cache = MemoryCache()
key = cache_key("foo", 4)
obj = {"key1": "value1"}
cache.put(key, obj)
obj["key2"] = "value2"
assert "key1" in cache.get(key)
assert "key2" not in cache.get(key)
assert "key2" in obj
def __init__(self, interconnect, source="usa_tamu", engine="REISE"):
"""Constructor."""
if not isinstance(source, str):
raise TypeError("source must be a str")
if source not in self.SUPPORTED_MODELS and not source.endswith(".mat"):
raise ValueError(
f"Source must be one of {','.join(self.SUPPORTED_MODELS)} "
"or the path to a .mat file that represents a grid ")
if engine not in self.SUPPORTED_ENGINES:
raise ValueError(
f"Engine must be one of {','.join(self.SUPPORTED_ENGINES)}")
key = cache_key(interconnect, source)
cached = _cache.get(key)
if cached is not None:
network = cached
elif source == "usa_tamu":
network = TAMU(interconnect)
elif source == "hifld":
network = HIFLD(interconnect)
elif os.path.splitext(source)[1] == ".mat":
if engine == "REISE":
network = FromREISE(source)
elif engine == "REISE.jl":
network = FromREISEjl(source)
else:
raise ValueError(f"Unknown source: {source}")
network.build()
self.data_loc = network.data_loc
self.interconnect = network.interconnect
self.zone2id = network.zone2id
self.id2zone = network.id2zone
self.sub = network.sub
self.plant = network.plant
self.gencost = network.gencost
self.dcline = network.dcline
self.bus2sub = network.bus2sub
self.bus = network.bus
self.branch = network.branch
self.storage = network.storage
self.grid_model = network.grid_model
self.model_immutables = network.model_immutables
_cache.put(key, network)
def __init__(self, interconnect, source="usa_tamu", engine="REISE"):
"""Constructor."""
if not isinstance(source, str):
raise TypeError("source must be a str")
if source not in self.SUPPORTED_MODELS and not source.endswith(".mat"):
raise ValueError(
f"Source must be one of {','.join(self.SUPPORTED_MODELS)} "
"or the path to a .mat file that represents a grid "
)
if engine not in self.SUPPORTED_ENGINES:
raise ValueError(
f"Engine must be one of {','.join(self.SUPPORTED_ENGINES)}"
)
key = cache_key(interconnect, source)
cached = _cache.get(key)
if cached is not None:
data = cached
elif source == "usa_tamu":
data = TAMU(interconnect)
elif os.path.splitext(source)[1] == ".mat":
if engine == "REISE":
data = FromREISE(source)
elif engine == "REISE.jl":
data = FromREISEjl(source)
self.data_loc = data.data_loc
self.interconnect = data.interconnect
self.zone2id = data.zone2id
self.id2zone = data.id2zone
self.sub = data.sub
self.plant = data.plant
self.gencost = data.gencost
self.dcline = data.dcline
self.bus2sub = data.bus2sub
self.bus = data.bus
self.branch = data.branch
self.storage = data.storage
_cache.put(key, self)
self.grid_model = self._get_grid_model()
self.model_immutables = ModelImmutables(self.grid_model)
def get_data(self, scenario_info, field_name):
"""Returns data either from server or local directory.
:param dict scenario_info: scenario information.
:param str field_name: *'demand'*, *'hydro'*, *'solar'*, *'wind'*,
*'ct'* or *'grid'*.
:return: (*pandas.DataFrame*, *dict*, or *str*) --
demand, hydro, solar or wind as a data frame, change table as a
dictionary, or the path to a matfile enclosing the grid data.
:raises FileNotFoundError: if file not found on local machine.
"""
self._check_field(field_name)
print("--> Loading %s" % field_name)
ext = self.file_extension[field_name]
if field_name in profile_kind:
version = scenario_info["base_" + field_name]
file_name = field_name + "_" + version + "." + ext
from_dir = posixpath.join(
server_setup.BASE_PROFILE_DIR, scenario_info["grid_model"]
)
else:
file_name = scenario_info["id"] + "_" + field_name + "." + ext
from_dir = server_setup.INPUT_DIR
filepath = os.path.join(server_setup.LOCAL_DIR, from_dir, file_name)
key = cache_key(filepath)
cached = _cache.get(key)
if cached is not None:
return cached
try:
data = _read_data(filepath)
except FileNotFoundError:
print(
"%s not found in %s on local machine"
% (file_name, server_setup.LOCAL_DIR)
)
self.data_access.copy_from(file_name, from_dir)
data = _read_data(filepath)
_cache.put(key, data)
return data
def __init__(self, interconnect, source="usa_tamu", engine="REISE"):
"""Constructor."""
if not isinstance(source, str):
raise TypeError("source must be a str")
supported_engines = {"REISE", "REISE.jl"}
if engine not in supported_engines:
raise ValueError(
f"Engine must be one of {','.join(supported_engines)}")
try:
self.model_immutables = ModelImmutables(source)
except ValueError:
self.model_immutables = ModelImmutables(
_get_grid_model_from_scenario_list(source))
key = cache_key(interconnect, source)
cached = _cache.get(key)
if cached is not None:
data = cached
elif source == "usa_tamu":
data = TAMU(interconnect)
elif os.path.splitext(source)[1] == ".mat":
if engine == "REISE":
data = FromREISE(source)
elif engine == "REISE.jl":
data = FromREISEjl(source)
self.data_loc = data.data_loc
self.interconnect = data.interconnect
self.zone2id = data.zone2id
self.id2zone = data.id2zone
self.sub = data.sub
self.plant = data.plant
self.gencost = data.gencost
self.dcline = data.dcline
self.bus2sub = data.bus2sub
self.bus = data.bus
self.branch = data.branch
self.storage = data.storage
_cache.put(key, self)
def get_data(self, scenario_info, field_name):
"""Returns data either from server or local directory.
:param dict scenario_info: scenario information.
:param str field_name: *'demand'*, *'hydro'*, *'solar'*, *'wind'*,
*'ct'* or *'grid'*.
:return: (*pandas.DataFrame*, *dict*, or *str*) --
demand, hydro, solar or wind as a data frame, change table as a
dictionary, or the path to a matfile enclosing the grid data.
:raises FileNotFoundError: if file not found on local machine.
"""
_check_field(field_name)
print("--> Loading %s" % field_name)
if field_name in profile_kind:
helper = ProfileHelper
else:
helper = InputHelper(self.data_access)
file_name, from_dir = helper.get_file_components(scenario_info, field_name)
filepath = os.path.join(server_setup.LOCAL_DIR, *from_dir, file_name)
key = cache_key(filepath)
cached = _cache.get(key)
if cached is not None:
return cached
try:
data = _read_data(filepath)
except FileNotFoundError:
print(
"%s not found in %s on local machine"
% (file_name, server_setup.LOCAL_DIR)
)
helper.download_file(file_name, from_dir)
data = _read_data(filepath)
_cache.put(key, data)
return data
def test_mem_cache_get_returns_copy():
cache = MemoryCache()
key = cache_key("foo", 4)
obj = {"key1": 42}
cache.put(key, obj)
assert id(cache.get(key)) != id(obj)
def test_mem_cache_put_dict():
cache = MemoryCache()
key = cache_key(["foo", "bar"], 4, "other")
obj = {"key1": 42}
cache.put(key, obj)
assert cache.get(key) == obj
def test_cache_key_distinct_types():
assert cache_key(4) != cache_key("4")
def test_cache_key_unsupported_type():
with pytest.raises(ValueError):
cache_key(object())
def test_no_collision():
key1 = cache_key([["foo"], ["bar"]])
key2 = cache_key([[["foo"], ["bar"]]])
key3 = cache_key([["foo"], "bar"])
keys = [key1, key2, key3]
assert len(keys) == len(set(keys))