def create_data_object(self):
"""restore data_object from cache
Returns:
data_object (DataObject)
"""
if self.configuration.config_metadata and "data_object" in self.configuration.config_metadata:
cfg = self.configuration.config_metadata["data_object"]
if "read_from_cache" in cfg and cfg["read_from_cache"]:
# we can assume that read_filename exists due to configuration checks
filename = cfg["read_filename"]
assert os.path.exists(filename)
logging.info("Reading DataObject from cache " + filename)
return DataObject.read_from_cache(filename)
data_object = DataObject(self.configuration)
return data_object
def test_caching():
config = {
"implementation_config": {
"reader_config": {
"csv_reader": {
"class": "CsvReader",
"filename": "test/minimal.csv",
"destinations": [],
}
}
}
}
configuration = Configuration(None, is_dict_config=True, dict_config=config)
data_object = DataObject(configuration)
writer = CsvReader(configuration, "csv_reader")
data_object.add(writer, "some_data")
filename = "test_data_object_cache.pkl"
if os.path.exists(filename):
os.remove(filename)
data_object.write_to_cache(filename)
assert os.path.exists(filename)
restored_data_object = DataObject.read_from_cache(filename)
assert isinstance(restored_data_object, DataObject)
assert (
restored_data_object.get("csv_reader", rtype=DataObjectResponseType.VALUE.value)
== "some_data"
)
if os.path.exists(filename):
os.remove(filename)
