def __exit__(self, etype, *args):
if etype is None:
step = core.to_execution_step(self)
step.RunEveryMillis(self.interval_ms)
if self._net:
self._net.add_attribute(Task.REPORT_STEP, step)
else:
TaskGroup.current().report_step(
step, interval_ms=self.interval_ms)
NetBuilder.__exit__(self, etype, *args)
def __exit__(self, etype, *args):
if etype is None:
step = core.to_execution_step(self)
step.RunEveryMillis(self.interval_ms)
if self._net:
self._net.add_attribute(Task.REPORT_STEP, step)
else:
TaskGroup.current().report_step(
step, interval_ms=self.interval_ms)
NetBuilder.__exit__(self, etype, *args)
def _pipe_step(input,
output=None,
num_threads=1,
processor=None,
name=None,
capacity=None,
group=None,
final_outputs=None):
"""
"""
group = TaskGroup.current(group)
if name is None:
name = 'processor:%d' % group.num_registered_tasks()
if isinstance(input, Reader):
reader = input
elif hasattr(input, 'reader'):
reader = input.reader()
else:
raise ValueError('in must be a reader, queue or streaam.')
if processor is not None:
reader = ProcessingReader(reader, processor)
if num_threads == 0:
assert output is None
return reader, None
global_exit_net = core.Net(name + '_producer_global_exit')
global_init_net = core.Net(name + '_producer_global_init')
out_queue = None
writer = None
reader.setup_ex(global_init_net, global_exit_net)
steps = []
for thread_id in range(num_threads):
init_net = core.Net(name + "_init_net_%d" % thread_id)
exit_net = core.Net(name + "_exit_net_%d" % thread_id)
read_nets, status, rec = reader.read_record_ex(init_net, exit_net)
if rec is not None:
if writer is None:
out_queue, writer = _init_output(output, capacity,
global_init_net,
global_exit_net)
write_nets, _ = writer.write_record_ex(rec, init_net, exit_net,
status)
else:
write_nets = []
step = core.execution_step(name + "_thread_%d" % thread_id, [
core.execution_step(name + "_init_step", init_net),
core.execution_step(name + "_worker_step",
list(read_nets) + list(write_nets),
should_stop_blob=status),
core.execution_step(name + "_exit_step", exit_net)
])
steps.append(step)
step = core.execution_step("sender_step", [
core.execution_step('init_step', global_init_net),
core.execution_step("sender_steps", steps, concurrent_substeps=True),
core.execution_step('finish_step', global_exit_net),
])
return out_queue, step
def _pipe_step(
input, output=None, num_threads=1, processor=None, name=None,
capacity=None, group=None, final_outputs=None):
"""
"""
group = TaskGroup.current(group)
if name is None:
name = 'processor:%d' % group.num_registered_tasks()
if isinstance(input, Reader):
reader = input
elif hasattr(input, 'reader'):
reader = input.reader()
else:
raise ValueError('in must be a reader, queue or streaam.')
if processor is not None:
reader = ProcessingReader(reader, processor)
if num_threads == 0:
assert output is None
return reader, None
global_exit_net = core.Net(name + '_producer_global_exit')
global_init_net = core.Net(name + '_producer_global_init')
out_queue = None
writer = None
reader.setup_ex(global_init_net, global_exit_net)
steps = []
for thread_id in range(num_threads):
init_net = core.Net(name + "_init_net_%d" % thread_id)
exit_net = core.Net(name + "_exit_net_%d" % thread_id)
read_nets, status, rec = reader.read_record_ex(init_net, exit_net)
if rec is not None:
if writer is None:
out_queue, writer = _init_output(
output, capacity, global_init_net, global_exit_net)
write_nets, _ = writer.write_record_ex(
rec, init_net, exit_net, status)
else:
write_nets = []
step = core.execution_step(
name + "_thread_%d" % thread_id, [
core.execution_step(name + "_init_step", init_net),
core.execution_step(
name + "_worker_step",
list(read_nets) + list(write_nets),
should_stop_blob=status
), core.execution_step(name + "_exit_step", exit_net)
]
)
steps.append(step)
step = core.execution_step(
"sender_step", [
core.execution_step('init_step', global_init_net),
core.execution_step(
"sender_steps", steps, concurrent_substeps=True),
core.execution_step('finish_step', global_exit_net),
])
return out_queue, step