def test_Metadata_constructor(dataspace): # noqa: F811 my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples metadata = datablock.Metadata(my_tm["taskmanager_id"]) assert metadata.data["taskmanager_id"] == my_tm["taskmanager_id"] genTime = 1.0 missCount = 3 state = "START_BACKUP" metadata = datablock.Metadata( my_tm["taskmanager_id"], state=state, generation_id=dataspace.get_last_generation_id( my_tm["name"], my_tm["taskmanager_id"]), generation_time=genTime, missed_update_count=missCount, ) assert metadata.data["taskmanager_id"] == my_tm["taskmanager_id"] assert metadata.data["state"] == state assert metadata.data["generation_id"] == dataspace.get_last_generation_id( my_tm["name"], my_tm["taskmanager_id"]) assert metadata.data["generation_time"] == genTime assert metadata.data["missed_update_count"] == missCount
def test_Metadata_constructor(self): dataproduct = self.data["dataproduct"][0] metadata = datablock.Metadata(dataproduct["taskmanager_id"]) self.assertEqual(metadata.data["taskmanager_id"], dataproduct["taskmanager_id"]) genTime = 1.0 missCount = 3 state = "START_BACKUP" metadata = datablock.Metadata(dataproduct["taskmanager_id"], state=state, generation_id=int( dataproduct["generation_id"]), generation_time=genTime, missed_update_count=missCount) self.assertEqual(metadata.data["taskmanager_id"], dataproduct["taskmanager_id"]) self.assertEqual(metadata.data["state"], state) self.assertEqual(metadata.data["generation_id"], int(dataproduct["generation_id"])) self.assertEqual(metadata.data["generation_time"], genTime) self.assertEqual(metadata.data["missed_update_count"], missCount) with self.assertRaises(datablock.InvalidMetadataError): metadata = datablock.Metadata(dataproduct["taskmanager_id"], "INVALID_STATE")
def test_DataBlock_key_management(dataspace): # noqa: F811 my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples header = datablock.Header(my_tm["taskmanager_id"]) metadata = datablock.Metadata( my_tm["taskmanager_id"], generation_id=dataspace.get_last_generation_id( my_tm["name"], my_tm["taskmanager_id"]), ) dblock = datablock.DataBlock(dataspace, my_tm["name"], my_tm["taskmanager_id"]) # test with automatic metadata and string value dblock.put("example_test_key", "example_test_value", header) assert "example_test_key" in dblock.keys() assert "example_test_key" in dblock assert dblock.get("example_test_key") == "example_test_value" # Test product-retriever interface retriever = datablock.ProductRetriever("example_test_key", None, None) assert retriever(dblock) == "example_test_value" assert ( str(retriever) == "Product retriever for {'name': 'example_test_key', 'type': None, 'creator': None}" ) # test new key with manual metadata and dict value newDict = {"subKey": "newValue"} dblock.put("new_example_test_key", newDict, header, metadata) assert dblock["new_example_test_key"] == newDict
def test_DataBlock_get_metadata(self): dataproduct = self.data["dataproduct"][0] header = datablock.Header(dataproduct["taskmanager_id"]) metadata = datablock.Metadata(dataproduct["taskmanager_id"], generation_id=int(dataproduct["generation_id"])) self.datablock.put(dataproduct["key"], dataproduct["value"], header, metadata) self.assertEqual(metadata, self.datablock.get_metadata(dataproduct["key"]))
def data_block_put(self, data, header, data_block): """ Put data into data block :type data: :obj:`dict` :arg data: key, value pairs :type header: :obj:`~datablock.Header` :arg header: data header :type data_block: :obj:`~datablock.DataBlock` :arg data_block: data block """ if not isinstance(data, dict): self.logger.error( f"data_block put expecting {dict} type, got {type(data)}") return self.logger.debug(f"data_block_put {data}") with data_block.lock: # This is too long to find, so im leaving it here to make it obvious whats going on # you'll want to update it eventually. # metadata = datablock.Metadata(data_block.component_manager_id, metadata = datablock.Metadata( data_block.taskmanager_id, state="END_CYCLE", generation_id=data_block.generation_id) for key, product in data.items(): data_block.put(key, product, header, metadata=metadata)
def test_Metadata_set_state(self): dataproduct = self.data["dataproduct"][0] metadata = datablock.Metadata(dataproduct["taskmanager_id"]) state = "START_BACKUP" metadata.set_state(state) self.assertEqual(metadata.data["state"], state) with self.assertRaises(datablock.InvalidMetadataError): metadata.set_state("INVALID_STATE")
def test_Metadata_set_state(dataspace): # noqa: F811 my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples metadata = datablock.Metadata(my_tm["taskmanager_id"]) state = "START_BACKUP" metadata.set_state(state) assert metadata.data["state"] == state with pytest.raises(datablock.InvalidMetadataError): metadata.set_state("INVALID_STATE")
def test_DataBlock_get_metadata(dataspace): # noqa: F811 my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples header = datablock.Header(my_tm["taskmanager_id"]) metadata = datablock.Metadata( my_tm["taskmanager_id"], generation_id=dataspace.get_last_generation_id( my_tm["name"], my_tm["taskmanager_id"]), ) dblock = datablock.DataBlock(dataspace, my_tm["name"], my_tm["taskmanager_id"]) dblock.put("example_test_key", "example_test_value", header, metadata) assert metadata == dblock.get_metadata("example_test_key")
def test_DataBlock_duplicate(self): dataproduct = self.data["dataproduct"][0] header = datablock.Header(dataproduct["taskmanager_id"]) metadata = datablock.Metadata(dataproduct["taskmanager_id"], generation_id=int(dataproduct["generation_id"])) self.datablock.put(dataproduct["key"], dataproduct["value"], header, metadata) dblock = self.datablock.duplicate() self.assertEqual(dblock.taskmanager_id, self.datablock.taskmanager_id) self.assertEqual(dblock.generation_id + 1, self.datablock.generation_id) self.assertEqual(dblock.sequence_id, self.datablock.sequence_id) self.assertEqual(dblock._keys, self.datablock._keys) for key in self.datablock._keys: self.assertEqual(dblock[key], self.datablock[key])
def data_block_put(self, data, header, data_block): """ Put data into data block :type data: :obj:`dict` :arg data: key, value pairs :type header: :obj:`~datablock.Header` :arg header: data header :type data_block: :obj:`~datablock.DataBlock` :arg data_block: data block """ if not isinstance(data, dict): logging.getLogger().error( f'data_block put expecting {dict} type, got {type(data)}') return logging.getLogger().debug(f'data_block_put {data}') with data_block.lock: metadata = datablock.Metadata( data_block.taskmanager_id, state='END_CYCLE', generation_id=data_block.generation_id) for key, product in data.items(): data_block.put(key, product, header, metadata=metadata)
def data_block_put(self, data, header, data_block): """ Put data into data block :type data: :obj:`dict` :arg data: key, value pairs :type header: :obj:`~datablock.Header` :arg header: data header :type data_block: :obj:`~datablock.DataBlock` :arg data_block: data block """ if not isinstance(data, dict): logging.getLogger().error( 'data_block put expecting %s type, got %s', dict, type(data)) return logging.getLogger().debug('data_block_put %s', data) with data_block.lock: for k in data: metadata = datablock.Metadata( data_block.taskmanager_id, generation_id=data_block.generation_id) metadata.set_state('END_CYCLE') data_block.put(k, data[k], header, metadata=metadata)