def test_Metadata_constructor(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
metadata = datablock.Metadata(my_tm["taskmanager_id"])
assert metadata.data["taskmanager_id"] == my_tm["taskmanager_id"]
genTime = 1.0
missCount = 3
state = "START_BACKUP"
metadata = datablock.Metadata(
my_tm["taskmanager_id"],
state=state,
generation_id=dataspace.get_last_generation_id(
my_tm["name"], my_tm["taskmanager_id"]),
generation_time=genTime,
missed_update_count=missCount,
)
assert metadata.data["taskmanager_id"] == my_tm["taskmanager_id"]
assert metadata.data["state"] == state
assert metadata.data["generation_id"] == dataspace.get_last_generation_id(
my_tm["name"], my_tm["taskmanager_id"])
assert metadata.data["generation_time"] == genTime
assert metadata.data["missed_update_count"] == missCount
def test_Metadata_constructor(self):
dataproduct = self.data["dataproduct"][0]
metadata = datablock.Metadata(dataproduct["taskmanager_id"])
self.assertEqual(metadata.data["taskmanager_id"],
dataproduct["taskmanager_id"])
genTime = 1.0
missCount = 3
state = "START_BACKUP"
metadata = datablock.Metadata(dataproduct["taskmanager_id"],
state=state,
generation_id=int(
dataproduct["generation_id"]),
generation_time=genTime,
missed_update_count=missCount)
self.assertEqual(metadata.data["taskmanager_id"],
dataproduct["taskmanager_id"])
self.assertEqual(metadata.data["state"], state)
self.assertEqual(metadata.data["generation_id"],
int(dataproduct["generation_id"]))
self.assertEqual(metadata.data["generation_time"], genTime)
self.assertEqual(metadata.data["missed_update_count"], missCount)
with self.assertRaises(datablock.InvalidMetadataError):
metadata = datablock.Metadata(dataproduct["taskmanager_id"],
"INVALID_STATE")
def test_DataBlock_key_management(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
metadata = datablock.Metadata(
my_tm["taskmanager_id"],
generation_id=dataspace.get_last_generation_id(
my_tm["name"], my_tm["taskmanager_id"]),
)
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
# test with automatic metadata and string value
dblock.put("example_test_key", "example_test_value", header)
assert "example_test_key" in dblock.keys()
assert "example_test_key" in dblock
assert dblock.get("example_test_key") == "example_test_value"
# Test product-retriever interface
retriever = datablock.ProductRetriever("example_test_key", None, None)
assert retriever(dblock) == "example_test_value"
assert (
str(retriever) ==
"Product retriever for {'name': 'example_test_key', 'type': None, 'creator': None}"
)
# test new key with manual metadata and dict value
newDict = {"subKey": "newValue"}
dblock.put("new_example_test_key", newDict, header, metadata)
assert dblock["new_example_test_key"] == newDict
def test_DataBlock_get_metadata(self):
dataproduct = self.data["dataproduct"][0]
header = datablock.Header(dataproduct["taskmanager_id"])
metadata = datablock.Metadata(dataproduct["taskmanager_id"], generation_id=int(dataproduct["generation_id"]))
self.datablock.put(dataproduct["key"], dataproduct["value"], header, metadata)
self.assertEqual(metadata, self.datablock.get_metadata(dataproduct["key"]))
def data_block_put(self, data, header, data_block):
"""
Put data into data block
:type data: :obj:`dict`
:arg data: key, value pairs
:type header: :obj:`~datablock.Header`
:arg header: data header
:type data_block: :obj:`~datablock.DataBlock`
:arg data_block: data block
"""
if not isinstance(data, dict):
self.logger.error(
f"data_block put expecting {dict} type, got {type(data)}")
return
self.logger.debug(f"data_block_put {data}")
with data_block.lock:
# This is too long to find, so im leaving it here to make it obvious whats going on
# you'll want to update it eventually.
# metadata = datablock.Metadata(data_block.component_manager_id,
metadata = datablock.Metadata(
data_block.taskmanager_id,
state="END_CYCLE",
generation_id=data_block.generation_id)
for key, product in data.items():
data_block.put(key, product, header, metadata=metadata)
def test_Metadata_set_state(self):
dataproduct = self.data["dataproduct"][0]
metadata = datablock.Metadata(dataproduct["taskmanager_id"])
state = "START_BACKUP"
metadata.set_state(state)
self.assertEqual(metadata.data["state"], state)
with self.assertRaises(datablock.InvalidMetadataError):
metadata.set_state("INVALID_STATE")
def test_Metadata_set_state(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
metadata = datablock.Metadata(my_tm["taskmanager_id"])
state = "START_BACKUP"
metadata.set_state(state)
assert metadata.data["state"] == state
with pytest.raises(datablock.InvalidMetadataError):
metadata.set_state("INVALID_STATE")
def test_DataBlock_get_metadata(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
metadata = datablock.Metadata(
my_tm["taskmanager_id"],
generation_id=dataspace.get_last_generation_id(
my_tm["name"], my_tm["taskmanager_id"]),
)
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
dblock.put("example_test_key", "example_test_value", header, metadata)
assert metadata == dblock.get_metadata("example_test_key")
def test_DataBlock_duplicate(self):
dataproduct = self.data["dataproduct"][0]
header = datablock.Header(dataproduct["taskmanager_id"])
metadata = datablock.Metadata(dataproduct["taskmanager_id"], generation_id=int(dataproduct["generation_id"]))
self.datablock.put(dataproduct["key"], dataproduct["value"], header, metadata)
dblock = self.datablock.duplicate()
self.assertEqual(dblock.taskmanager_id, self.datablock.taskmanager_id)
self.assertEqual(dblock.generation_id + 1, self.datablock.generation_id)
self.assertEqual(dblock.sequence_id, self.datablock.sequence_id)
self.assertEqual(dblock._keys, self.datablock._keys)
for key in self.datablock._keys:
self.assertEqual(dblock[key], self.datablock[key])
def data_block_put(self, data, header, data_block):
"""
Put data into data block
:type data: :obj:`dict`
:arg data: key, value pairs
:type header: :obj:`~datablock.Header`
:arg header: data header
:type data_block: :obj:`~datablock.DataBlock`
:arg data_block: data block
"""
if not isinstance(data, dict):
logging.getLogger().error(
f'data_block put expecting {dict} type, got {type(data)}')
return
logging.getLogger().debug(f'data_block_put {data}')
with data_block.lock:
metadata = datablock.Metadata(
data_block.taskmanager_id,
state='END_CYCLE',
generation_id=data_block.generation_id)
for key, product in data.items():
data_block.put(key, product, header, metadata=metadata)
def data_block_put(self, data, header, data_block):
"""
Put data into data block
:type data: :obj:`dict`
:arg data: key, value pairs
:type header: :obj:`~datablock.Header`
:arg header: data header
:type data_block: :obj:`~datablock.DataBlock`
:arg data_block: data block
"""
if not isinstance(data, dict):
logging.getLogger().error(
'data_block put expecting %s type, got %s', dict, type(data))
return
logging.getLogger().debug('data_block_put %s', data)
with data_block.lock:
for k in data:
metadata = datablock.Metadata(
data_block.taskmanager_id,
generation_id=data_block.generation_id)
metadata.set_state('END_CYCLE')
data_block.put(k, data[k], header, metadata=metadata)
