def test_Header_constructor(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
assert header.data["taskmanager_id"] == my_tm["taskmanager_id"]
createTime = 1.0
expirationTime = 3.0
scheduleTime = 5.0
creator = "creator"
schema = 1
header = datablock.Header(
my_tm["taskmanager_id"],
create_time=createTime,
expiration_time=expirationTime,
scheduled_create_time=scheduleTime,
creator=creator,
schema_id=schema,
)
assert header.data["taskmanager_id"] == my_tm["taskmanager_id"]
assert header.data["create_time"] == createTime
assert header.data["expiration_time"] == expirationTime
assert header.data["scheduled_create_time"] == scheduleTime
assert header.data["creator"] == creator
assert header.data["schema_id"] == schema
def test_Header_constructor(self):
dataproduct = self.data["dataproduct"][0]
header = datablock.Header(dataproduct["taskmanager_id"])
self.assertEqual(header.data["taskmanager_id"],
dataproduct["taskmanager_id"])
createTime = 1.0
expirationTime = 3.0
scheduleTime = 5.0
creator = "creator"
schema = 1
header = datablock.Header(dataproduct["taskmanager_id"],
create_time=createTime,
expiration_time=expirationTime,
scheduled_create_time=scheduleTime,
creator=creator,
schema_id=schema)
self.assertEqual(header.data["taskmanager_id"],
dataproduct["taskmanager_id"])
self.assertEqual(header.data["create_time"], createTime)
self.assertEqual(header.data["expiration_time"], expirationTime)
self.assertEqual(header.data["scheduled_create_time"], scheduleTime)
self.assertEqual(header.data["creator"], creator)
self.assertEqual(header.data["schema_id"], schema)
def test_DataBlock_get_metadata(self):
dataproduct = self.data["dataproduct"][0]
header = datablock.Header(dataproduct["taskmanager_id"])
metadata = datablock.Metadata(dataproduct["taskmanager_id"], generation_id=int(dataproduct["generation_id"]))
self.datablock.put(dataproduct["key"], dataproduct["value"], header, metadata)
self.assertEqual(metadata, self.datablock.get_metadata(dataproduct["key"]))
def test_DataBlock_to_str(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
expected = {
"taskmanager_id":
my_tm["taskmanager_id"],
"generation_id":
dataspace.get_last_generation_id(my_tm["name"],
my_tm["taskmanager_id"]),
"sequence_id":
len(dataspace.get_dataproducts(my_tm["sequence_id"])) + 1,
"keys": [
"example_test_key",
],
"dataproducts": {
"example_test_key": "example_test_value"
},
}
header = datablock.Header(my_tm["taskmanager_id"])
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
dblock.put("example_test_key", "example_test_value", header)
result = ast.literal_eval(str(dblock))
assert result == expected
def test_DataBlock_key_management(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
metadata = datablock.Metadata(
my_tm["taskmanager_id"],
generation_id=dataspace.get_last_generation_id(
my_tm["name"], my_tm["taskmanager_id"]),
)
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
# test with automatic metadata and string value
dblock.put("example_test_key", "example_test_value", header)
assert "example_test_key" in dblock.keys()
assert "example_test_key" in dblock
assert dblock.get("example_test_key") == "example_test_value"
# Test product-retriever interface
retriever = datablock.ProductRetriever("example_test_key", None, None)
assert retriever(dblock) == "example_test_value"
assert (
str(retriever) ==
"Product retriever for {'name': 'example_test_key', 'type': None, 'creator': None}"
)
# test new key with manual metadata and dict value
newDict = {"subKey": "newValue"}
dblock.put("new_example_test_key", newDict, header, metadata)
assert dblock["new_example_test_key"] == newDict
def run_transform(self, worker, data_block):
"""
Run a transform
:type worker: :obj:`~Worker`
:arg worker: Transform worker
:type data_block: :obj:`~datablock.DataBlock`
:arg data_block: data block
"""
consume_keys = list(worker.module_instance._consumes.keys())
self.logger.info("transform: %s expected keys: %s provided keys: %s",
worker.name, consume_keys, list(data_block.keys()))
self.logger.info("Run transform %s", worker.name)
try:
with TRANSFORM_RUN_HISTOGRAM.labels(self.name, worker.name).time():
data = worker.module_instance.transform(data_block)
self.logger.debug(f"transform returned {data}")
header = datablock.Header(data_block.taskmanager_id,
create_time=time.time(),
creator=worker.name)
self.data_block_put(data, header, data_block)
self.logger.info("transform put data")
TRANSFORM_RUN_GAUGE.labels(self.name,
worker.name).set_to_current_time()
except Exception: # pragma: no cover
self.logger.exception(f"exception from transform {worker.name} ")
self.take_offline()
def test_DataBlock_key_management(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
dblock.put("example_test_key", "example_test_value", header)
assert "example_test_key" in dblock.keys()
assert "example_test_key" in dblock
assert dblock.get("example_test_key") == "example_test_value"
# Test product-retriever interface
retriever = datablock.ProductRetriever("example_test_key", None, None)
assert retriever(dblock) == "example_test_value"
assert (
str(retriever) ==
"Product retriever for {'name': 'example_test_key', 'type': None, 'creator': None}"
)
# FIXME: The following behavior should be disallowed for data-integrity reasons!
# i.e. replacing a product name with a different value.
newDict = {"subKey": "newValue"}
dblock.put("example_test_key", newDict, header)
assert dblock["example_test_key"] == newDict
def test_DataBlock_key_management(self):
dataproduct = self.data["dataproduct"][0]
header = datablock.Header(dataproduct["taskmanager_id"])
self.datablock.put(dataproduct["key"], dataproduct["value"], header)
self.assertIn(dataproduct["key"], self.datablock.keys())
self.assertIn(dataproduct["key"], self.datablock)
self.assertEqual(self.datablock.get(dataproduct["key"]),
dataproduct["value"])
# Test product-retriever interface
retriever = datablock.ProductRetriever(dataproduct["key"], None, None)
assert retriever(self.datablock) == dataproduct["value"]
assert str(
retriever
) == "Product retriever for {'name': 'test_key1', 'type': None, 'creator': None}"
# FIXME: The following behavior should be disallowed for data-integrity reasons!
# i.e. replacing a product name with a different value.
newDict = {"subKey": "newValue"}
self.datablock.put(dataproduct["key"], newDict, header)
self.assertEqual(self.datablock[dataproduct["key"]], newDict)
with self.assertRaises(KeyError):
self.datablock["invalidKey"]
def test_DataBlock_get_taskmanagers(self):
taskmanager = self.data["taskmanager"][0]
dataproduct = self.data["dataproduct"][0]
header = datablock.Header(dataproduct["taskmanager_id"])
self.datablock.put(dataproduct["key"], dataproduct["value"], header)
tms = self.dataspace.get_taskmanagers()
self.assertEqual(taskmanager["taskmanager_id"], tms[0]["taskmanager_id"])
def run_transform(self, transform, data_block):
"""
Run a transform
:type transform: :obj:`~Worker`
:arg transform: source Worker
:type data_block: :obj:`~datablock.DataBlock`
:arg data_block: data block
"""
consume_keys = list(transform.worker._consumes.keys())
logging.getLogger().info(
'transform: %s expected keys: %s provided keys: %s',
transform.name, consume_keys, list(data_block.keys()))
logging.getLogger().info('run transform %s', transform.name)
try:
data = transform.worker.transform(data_block)
logging.getLogger().debug(f'transform returned {data}')
header = datablock.Header(data_block.taskmanager_id,
create_time=time.time(),
creator=transform.name)
self.data_block_put(data, header, data_block)
logging.getLogger().info('transform put data')
except Exception: # pragma: no cover
logging.getLogger().exception(
f'exception from transform {transform.name} ')
self.take_offline(data_block)
def test_DataBlock_get_taskmanager(self):
taskmanager = self.data["taskmanager"][0]
dataproduct = self.data["dataproduct"][0]
header = datablock.Header(dataproduct["taskmanager_id"])
self.datablock.put(dataproduct["key"], dataproduct["value"], header)
tid = self.datablock.get_taskmanager(taskmanager["name"])["taskmanager_id"]
self.assertEqual(taskmanager["taskmanager_id"], tid)
def test_DataBlock_get_header(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
dblock.put("example_test_key", "example_test_value", header)
assert header == dblock.get_header("example_test_key")
def test_DataBlock_no_key_by_name(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
dblock.put("example_test_key", "example_test_value", header)
with pytest.raises(KeyError):
dblock["no_such_key_exists"]
def run_cycle(body, message):
module_spec = body["source_module"]
module_name = body["class_name"]
data = body["data"]
assert data
self.logger.debug(f"Data received from {module_name}: {data}")
if not self.sources_have_run_once:
self.source_product_cache.update(**data)
missing_products = self.expected_source_products - set(
self.source_product_cache.keys())
if missing_products:
self.logger.info(
f"Waiting on more data (missing {missing_products})"
)
message.ack()
return
self.logger.info("All sources have executed at least once")
data = self.source_product_cache
self.sources_have_run_once = True
header = datablock.Header(self.data_block_t0.taskmanager_id,
create_time=time.time(),
creator=module_spec)
self.logger.info(f"Source {module_name} header done")
try:
self.data_block_put(data, header, self.data_block_t0)
except Exception: # pragma: no cover
self.logger.exception(
"Exception inserting data into the data block.")
self.logger.error(
f"Could not insert data from the following message\n{body}"
)
message.ack()
return
self.logger.info(f"Source {module_name} data block put done")
try:
self.decision_cycle()
with self.state.lock:
if not self.state.should_stop():
# If we are signaled to stop, don't override that state
# otherwise the last decision_cycle completed without error
self.state.set(State.STEADY)
CHANNEL_STATE_GAUGE.labels(self.name).set(
self.get_state_value())
except Exception: # pragma: no cover
self.logger.exception(
"Exception in the task manager main loop")
self.logger.error(
"Error occured. Task manager %s exits with state %s",
self.id, self.get_state_name())
message.ack()
def test_DataBlock_to_str(self):
dataproduct = self.data["dataproduct"][0]
header = datablock.Header(dataproduct["taskmanager_id"])
self.datablock.put(dataproduct["key"], dataproduct["value"], header)
result = str(self.datablock)
self.assertEqual(
result,
"{'taskamanger_id': '1', 'generation_id': 1, 'sequence_id': 2, "
"'keys': ['%s'], 'dataproducts': {'%s': '%s'}}" %
(dataproduct["key"], dataproduct["key"], dataproduct["value"]))
def test_DataBlock_mark_expired(dataspace): # noqa: F811
# mark_expired is just a stub in this case
# failure in a real implementation should raise an exception
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
dblock.put("example_test_key", "example_test_value", header)
assert dblock.mark_expired(1) is None
def test_DataBlock_get_dataproducts(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
dblock.put("example_test_key", "example_test_value", header)
products = dblock.get_dataproducts()
assert len(products) == 1
assert products[0]["key"] == "example_test_key"
assert products[0]["value"] == "example_test_value"
def run_cycle(self, messages):
for name, msg_body in messages.items():
module_spec = msg_body["source_module"]
module_name = msg_body["class_name"]
data = msg_body["data"]
assert data
if data is State.SHUTDOWN:
self.logger.info(
f"Channel {self.name} has received shutdown flag from source {module_spec} (class {module_name})"
)
self.take_offline()
return
assert isinstance(data, dict)
self.logger.debug(f"Data received from {module_name}: {data}")
data_to_process = self.source_product_cache.update(data)
if data_to_process is None:
return
header = datablock.Header(self.data_block_t0.taskmanager_id,
create_time=time.time(),
creator=module_spec)
self.logger.info(f"Source {module_name} header done")
try:
self.data_block_put(data_to_process, header,
self.data_block_t0)
except Exception: # pragma: no cover
self.logger.exception(
"Exception inserting data into the data block.")
self.logger.error(
f"Could not insert data from the following message\n{msg_body}"
)
return
self.logger.info(f"Source {module_name} data block put done")
try:
self.decision_cycle()
with self.state.lock:
if not self.state.should_stop():
# If we are signaled to stop, don't override that state
# otherwise the last decision_cycle completed without error
self.state.set(State.STEADY)
CHANNEL_STATE_GAUGE.labels(self.name).set(
self.get_state_value())
except Exception: # pragma: no cover
self.logger.exception("Exception in the task manager main loop")
self.logger.error(
"Error occured. Task manager %s exits with state %s", self.id,
self.get_state_name())
def run_logic_engine(self, data_block=None):
"""
Run Logic Engine.
:type data_block: :obj:`~datablock.DataBlock`
:arg data_block: data block
"""
le_list = []
if not data_block:
return
try:
for le in self.channel.le_s:
logging.getLogger().info('run logic engine %s',
self.channel.le_s[le].name)
logging.getLogger().debug('run logic engine %s %s',
self.channel.le_s[le].name,
data_block)
rc = self.channel.le_s[le].worker.evaluate(data_block)
le_list.append(rc)
logging.getLogger().info('run logic engine %s done',
self.channel.le_s[le].name)
logging.getLogger().info(
'logic engine %s generated newfacts: %s',
self.channel.le_s[le].name,
rc['newfacts'].to_dict(orient='records'))
logging.getLogger().info(
'logic engine %s generated actions: %s',
self.channel.le_s[le].name, rc['actions'])
# Add new facts to the datablock
# Add empty dataframe if nothing is available
if le_list:
all_facts = pandas.concat([i['newfacts'] for i in le_list],
ignore_index=True)
else:
logging.getLogger().info(
'Logic engine(s) did not return any new facts')
all_facts = pandas.DataFrame()
data = {'de_logicengine_facts': all_facts}
t = time.time()
header = datablock.Header(data_block.taskmanager_id,
create_time=t,
creator='logicengine')
self.data_block_put(data, header, data_block)
except Exception: # pragma: no cover
logging.getLogger().exception("Unexpected error!")
raise
else:
return le_list
def run_transform(self, transform, data_block):
"""
Run a transform
:type transform: :obj:`~Worker`
:arg transform: source Worker
:type data_block: :obj:`~datablock.DataBlock`
:arg data_block: data block
"""
data_to = self.channel.task_manager.get('data_TO', TRANSFORMS_TO)
consume_keys = transform.worker.consumes()
logging.getLogger().info(
'transform: %s expected keys: %s provided keys: %s',
transform.name, consume_keys, list(data_block.keys()))
loop_counter = 0
while True:
# Check if data is ready
if set(consume_keys) <= set(data_block.keys()):
# data is ready - may run transform()
logging.getLogger().info('run transform %s', transform.name)
try:
with data_block.lock:
data = transform.worker.transform(data_block)
logging.getLogger().debug('transform returned %s', data)
t = time.time()
header = datablock.Header(data_block.taskmanager_id,
create_time=t,
creator=transform.name)
self.data_block_put(data, header, data_block)
logging.getLogger().info('transform put data')
except Exception as e:
log_exception(
logging.getLogger(),
'exception from transform {} : {}'.format(
transform.name, e))
self.offline_task_manager(data_block)
break
else:
s = transform.stop_running.wait(1)
if s:
logging.getLogger().info(
'received stop_running signal for %s', transform.name)
break
loop_counter += 1
if loop_counter == data_to:
logging.getLogger().info(
'transform %s did not get consumes data in %s seconds. Exiting',
transform.name, data_to)
break
transform.data_updated.set()
def test_DataBlock_duplicate(self):
dataproduct = self.data["dataproduct"][0]
header = datablock.Header(dataproduct["taskmanager_id"])
metadata = datablock.Metadata(dataproduct["taskmanager_id"], generation_id=int(dataproduct["generation_id"]))
self.datablock.put(dataproduct["key"], dataproduct["value"], header, metadata)
dblock = self.datablock.duplicate()
self.assertEqual(dblock.taskmanager_id, self.datablock.taskmanager_id)
self.assertEqual(dblock.generation_id + 1, self.datablock.generation_id)
self.assertEqual(dblock.sequence_id, self.datablock.sequence_id)
self.assertEqual(dblock._keys, self.datablock._keys)
for key in self.datablock._keys:
self.assertEqual(dblock[key], self.datablock[key])
def run_logic_engine(self, data_block=None):
"""
Run Logic Engine.
:type data_block: :obj:`~datablock.DataBlock`
:arg data_block: data block
"""
le_list = []
if not data_block:
return
try:
for le in self.channel.le_s:
self.logger.info("run logic engine %s",
self.channel.le_s[le].name)
self.logger.debug("run logic engine %s %s",
self.channel.le_s[le].name, data_block)
rc = self.channel.le_s[le].worker.evaluate(data_block)
le_list.append(rc)
self.logger.info("run logic engine %s done",
self.channel.le_s[le].name)
self.logger.info(
"logic engine %s generated newfacts: %s",
self.channel.le_s[le].name,
rc["newfacts"].to_dict(orient="records"),
)
self.logger.info("logic engine %s generated actions: %s",
self.channel.le_s[le].name, rc["actions"])
# Add new facts to the datablock
# Add empty dataframe if nothing is available
if le_list:
all_facts = pd.concat([i["newfacts"] for i in le_list],
ignore_index=True)
else:
self.logger.info(
"Logic engine(s) did not return any new facts")
all_facts = pd.DataFrame()
data = {"de_logicengine_facts": all_facts}
t = time.time()
header = datablock.Header(data_block.taskmanager_id,
create_time=t,
creator="logicengine")
self.data_block_put(data, header, data_block)
except Exception: # pragma: no cover
self.logger.exception("Unexpected error!")
raise
else:
return le_list
def test_DataBlock_key_management_change_name(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
dblock.put("example_test_key", "example_test_value", header)
# FIXME: The following behavior should be disallowed for data-integrity reasons!
# i.e. replacing a product name from datablock.ProductRetriever with a
# different value.
newDict = {"subKey": "newValue"}
dblock.put("example_test_key", newDict, header)
assert dblock["example_test_key"] == newDict
def test_DataBlock_is_expired_with_key(dataspace): # noqa: F811
"""This test just validates the method/function exists.
The stub within our default code should be replaced
by a class inheriting from it.
That class should have more rational return types.
"""
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
dblock.put("example_test_key", "example_test_value", header)
assert dblock.is_expired(key="example_test_key") is None
def test_DataBlock_get_metadata(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
metadata = datablock.Metadata(
my_tm["taskmanager_id"],
generation_id=dataspace.get_last_generation_id(
my_tm["name"], my_tm["taskmanager_id"]),
)
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
dblock.put("example_test_key", "example_test_value", header, metadata)
assert metadata == dblock.get_metadata("example_test_key")
def run_transform(self, transform, data_block):
"""
Run a transform
:type transform: :obj:`~Worker`
:arg transform: source Worker
:type data_block: :obj:`~datablock.DataBlock`
:arg data_block: data block
"""
data_to = self.channel.task_manager.get('data_TO', _TRANSFORMS_TO)
consume_keys = list(transform.worker._consumes.keys())
logging.getLogger().info(
'transform: %s expected keys: %s provided keys: %s',
transform.name, consume_keys, list(data_block.keys()))
loop_counter = 0
while not self.state.should_stop():
# Check if data is ready
if set(consume_keys) <= set(data_block.keys()):
# data is ready - may run transform()
logging.getLogger().info('run transform %s', transform.name)
try:
with data_block.lock:
data = transform.worker.transform(data_block)
logging.getLogger().debug(f'transform returned {data}')
t = time.time()
header = datablock.Header(data_block.taskmanager_id,
create_time=t,
creator=transform.name)
self.data_block_put(data, header, data_block)
logging.getLogger().info('transform put data')
except Exception: # pragma: no cover
logging.getLogger().exception(
f'exception from transform {transform.name} ')
self.take_offline(data_block)
break
s = transform.stop_running.wait(1)
if s:
logging.getLogger().info(
f'received stop_running signal for {transform.name}')
break
loop_counter += 1
if loop_counter == data_to:
logging.getLogger().info(
f'transform {transform.name} did not get consumes data'
f'in {data_to} seconds. Exiting')
break
transform.data_updated.set()
def run_source(self, src):
"""
Get the data from source
and put it into the data block
:type src: :obj:`~Worker`
:arg src: source Worker
"""
# If task manager is in offline state, do not keep executing sources.
while not self.state.should_stop():
try:
logging.getLogger().info(f'Src {src.name} calling acquire')
data = src.worker.acquire()
Module.verify_products(src.worker, data)
logging.getLogger().info(f'Src {src.name} acquire retuned')
logging.getLogger().info(f'Src {src.name} filling header')
if data:
t = time.time()
header = datablock.Header(
self.data_block_t0.taskmanager_id,
create_time=t,
creator=src.module)
logging.getLogger().info(f'Src {src.name} header done')
self.data_block_put(data, header, self.data_block_t0)
logging.getLogger().info(
f'Src {src.name} data block put done')
else:
logging.getLogger().warning(
f'Src {src.name} acquire retuned no data')
src.run_counter += 1
src.data_updated.set()
logging.getLogger().info(
f'Src {src.name} {src.module} finished cycle')
except Exception:
logging.getLogger().exception(
f'Exception running source {src.name} ')
self.take_offline(self.data_block_t0)
if src.schedule > 0:
s = src.stop_running.wait(src.schedule)
if s:
logging.getLogger().info(
f'received stop_running signal for {src.name}')
break
else:
logging.getLogger().info(f'source {src.name} runs only once')
break
logging.getLogger().info(f'stopped {src.name}')
def run_transform(self, transform, data_block):
"""
Run a transform
:type transform: :obj:`~Worker`
:arg transform: source Worker
:type data_block: :obj:`~datablock.DataBlock`
:arg data_block: data block
"""
data_to = self.channel.task_manager.get('data_TO', TRANSFORMS_TO)
consume_keys = transform.worker.consumes()
self.logger.info('transform: %s expected keys: %s provided keys: %s' %
(transform.name, consume_keys, data_block.keys()))
loop_counter = 0
while 1:
# Check if data is ready
if set(consume_keys) <= set(data_block.keys()):
# data is ready - may run transform()
self.logger.info('run transform %s' % (transform.name, ))
try:
with data_block.lock:
data = transform.worker.transform(data_block)
self.logger.debug('transform returned %s' % (data, ))
t = time.time()
header = datablock.Header(data_block.taskmanager_id,
create_time=t,
creator=transform.name)
self.data_block_put(data, header, data_block)
self.logger.info('transform put data')
except Exception, detail:
self.logger.error('exception from %s: %s' %
(transform.name, detail))
self.offline_task_manager(data_block)
break
else:
s = transform.stop_running.wait(1)
if s:
self.logger.info("received stop_running signal for %s" %
(transform.name, ))
break
loop_counter += 1
if loop_counter == data_to:
self.logger.info(
"transform %s did not get consumes data in %s seconds. Exiting"
% (transform.name, data_to))
break
def run_source(self, src):
"""
Get the data from source
and put it into the data block
:type src: :obj:`~Worker`
:arg src: source Worker
"""
while True:
try:
logging.getLogger().info('Src %s calling acquire', src.name)
data = src.worker.acquire()
logging.getLogger().info('Src %s acquire retuned', src.name)
logging.getLogger().info('Src %s filling header', src.name)
if data:
t = time.time()
header = datablock.Header(
self.data_block_t0.taskmanager_id,
create_time=t,
creator=src.module)
logging.getLogger().info('Src %s header done', src.name)
self.data_block_put(data, header, self.data_block_t0)
logging.getLogger().info('Src %s data block put done',
src.name)
else:
logging.getLogger().warning(
'Src %s acquire retuned no data', src.name)
src.run_counter += 1
src.data_updated.set()
logging.getLogger().info('Src %s %s finished cycle', src.name,
src.module)
except Exception as e:
log_exception(
logging.getLogger(),
'Exception running source {} : {}'.format(src.name, e))
self.offline_task_manager(self.data_block_t0)
if src.schedule > 0:
s = src.stop_running.wait(src.schedule)
if s:
logging.getLogger().info(
'received stop_running signal for %s', src.name)
break
else:
logging.getLogger().info('source %s runs only once', src.name)
break
logging.getLogger().info('stopped %s', src.name)
def test_DataBlock_duplicate(dataspace): # noqa: F811
my_tm = dataspace.get_taskmanagers()[0] # fetch one of our loaded examples
header = datablock.Header(my_tm["taskmanager_id"])
dblock = datablock.DataBlock(dataspace, my_tm["name"],
my_tm["taskmanager_id"])
dblock.put("example_test_key", "example_test_value", header)
dblock_2 = dblock.duplicate()
assert dblock.taskmanager_id == dblock_2.taskmanager_id
assert dblock.generation_id == dblock_2.generation_id + 1
assert dblock.sequence_id == dblock_2.sequence_id
assert dblock._keys == dblock_2._keys
for key in dblock._keys:
assert dblock[key] == dblock_2[key]
