def _static_threads_task(name, group, final_outputs, reader, num_threads,
output, capacity):
node_name = str(Node.current())
profiler_name = "{0}/{1}/{2}/{3}/{4}".format(
node_name, "pipe", name,
processor_name(input) if input else "NoInput",
processor_name(output) if output else "NoOutput")
with Task(name=name, group=group, outputs=final_outputs) as task:
global_exit_net = core.Net('exit')
global_init_net = core.Net('init')
reader.setup_ex(global_init_net, global_exit_net)
out_queue = None
writer = None
steps = []
for thread_id in range(num_threads):
with NetBuilder(name='t:%d' % thread_id) as nb:
init_net = core.Net('init')
exit_net = core.Net('exit')
read_nets, status, rec = reader.read_record_ex(
init_net, exit_net)
init_net.ConstantFill([], [status],
shape=[],
value=False,
dtype=core.DataType.BOOL)
if rec is not None:
if writer is None:
# hack so that the out queue gets the right name prefix
# (otherwise they would be prefixed with the thread id)
with NetBuilder(_fullname=task.name):
out_queue, writer = _init_output(
output, capacity, global_init_net,
global_exit_net)
write_nets, _ = writer.write_record_ex(
rec, init_net, exit_net, status)
else:
write_nets = []
timer_start_net = core.Net('timer_start')
timer = timer_start_net.TimerBegin([],
counter_name=profiler_name)
timer_end_net = core.Net('timer_end')
timer_end_net.TimerEnd(timer, [])
ops.net(init_net)
ops.net(
core.execution_step('body',
[timer_start_net] + list(read_nets) +
list(write_nets) + [timer_end_net],
should_stop_blob=status))
ops.net(timer_end_net)
ops.net(exit_net)
steps.append(core.to_execution_step(nb))
ops.net(global_init_net)
ops.net(core.execution_step('body', steps, concurrent_substeps=True))
ops.net(global_exit_net)
return out_queue, task
def report_step(self, step=None, node=None, interval_ms=1000):
"""
Add a "report step" to this TaskGroup. This step will run repeatedly
every `interval_ms` milliseconds for the duration of the TaskGroup
execution on each of the nodes. It is guaranteed that this step
will be run at least once after every Task in the node has finished.
"""
step = core.to_execution_step(step)
step.RunEveryMillis(interval_ms)
self._report_steps.append((str(node or Node.current(node)), step))
def __exit__(self, etype, *args):
if etype is None:
step = core.to_execution_step(self)
step.RunEveryMillis(self.interval_ms)
if self._net:
self._net.add_attribute(Task.REPORT_STEP, step)
else:
TaskGroup.current().report_step(
step, interval_ms=self.interval_ms)
NetBuilder.__exit__(self, etype, *args)
def report_step(self, step=None, node=None, interval_ms=1000):
"""
Add a "report step" to this TaskGroup. This step will run repeatedly
every `interval_ms` milliseconds for the duration of the TaskGroup
execution on each of the nodes. It is guaranteed that this step
will be run at least once after every Task in the node has finished.
"""
step = core.to_execution_step(step)
step.RunEveryMillis(interval_ms)
self._report_steps.append((str(node or Node.current(node)), step))
def __exit__(self, etype, *args):
if etype is None:
step = core.to_execution_step(self)
step.RunEveryMillis(self.interval_ms)
if self._net:
self._net.add_attribute(Task.REPORT_STEP, step)
else:
TaskGroup.current().report_step(
step, interval_ms=self.interval_ms)
NetBuilder.__exit__(self, etype, *args)
def test_while_net(self):
with NetBuilder() as nb:
x = ops.Const(0)
y = ops.Const(0)
with ops.WhileNet():
with ops.Condition():
ops.Add([x, ops.Const(1)], [x])
ops.LT([x, ops.Const(7)])
ops.Add([x, y], [y])
plan = Plan('while_net_test')
plan.AddStep(to_execution_step(nb))
ws = workspace.C.Workspace()
ws.run(plan)
x_value = ws.blobs[str(x)].fetch()
y_value = ws.blobs[str(y)].fetch()
self.assertEqual(x_value, 7)
self.assertEqual(y_value, 21)
def test_while_net(self):
with NetBuilder() as nb:
x = ops.Const(0)
y = ops.Const(0)
with ops.WhileNet():
with ops.Condition():
ops.Add([x, ops.Const(1)], [x])
ops.LT([x, ops.Const(7)])
ops.Add([x, y], [y])
plan = Plan('while_net_test')
plan.AddStep(to_execution_step(nb))
ws = workspace.C.Workspace()
ws.run(plan)
x_value = ws.blobs[str(x)].fetch()
y_value = ws.blobs[str(y)].fetch()
self.assertEqual(x_value, 7)
self.assertEqual(y_value, 21)
def test_ops(self):
with NetBuilder() as nb:
y = _test_loop()
z, w, a, b = _test_outer()
p = _test_if(ops.Const(75))
q = _test_if(ops.Const(25))
plan = Plan('name')
plan.AddStep(to_execution_step(nb))
ws = workspace.C.Workspace()
ws.run(plan)
expected = [
(y, 5),
(z, False),
(w, True),
(a, False),
(b, False),
(p, 2),
(q, 3),
]
for b, expected in expected:
actual = ws.blobs[str(b)].fetch()
self.assertEquals(actual, expected)
def test_ops(self):
with NetBuilder() as nb:
y = _test_loop()
z, w, a, b = _test_outer()
p = _test_if(ops.Const(75))
q = _test_if(ops.Const(25))
plan = Plan('name')
plan.AddStep(to_execution_step(nb))
ws = workspace.C.Workspace()
ws.run(plan)
expected = [
(y, 5),
(z, False),
(w, True),
(a, False),
(b, False),
(p, 2),
(q, 3),
]
for b, expected in expected:
actual = ws.blobs[str(b)].fetch()
self.assertEquals(actual, expected)
with NetBuilder() as nb:
ops.Const(0.0, blob_out="zero")
ops.Const(1.0, blob_out="one")
ops.Const(0.5, blob_out="x")
ops.Const(0.0, blob_out="y")
with ops.IfNet(ops.GT(["x", "zero"])):
ops.Copy("one", "y")
with ops.Else():
ops.Copy("zero", "y")
# Note the usage of NetBuilder's ops.IfNet and ops.Else calls: ops.IfNet accepts a blob reference or blob name as an input, it expects an input blob to have a scalar value convertible to bool, also note that optional ops.Else is at the same level as ops.IfNet and immediately follows corresponding ops.IfNet. Let's execute resulting net (execution step) and check values of blobs.
# In[3]:
plan = Plan('if_net_test')
plan.AddStep(to_execution_step(nb))
ws = workspace.C.Workspace()
ws.run(plan)
print('x = ', ws.blobs["x"].fetch())
print('y = ', ws.blobs["y"].fetch())
# Before going further, it's important to understand the semantics of execution blocks ('then' and 'else' branches in the example above), i.e. handling of reads and writes into global (defined outside of the block) and local (defined inside the block) blobs.
# NetBuilder's uses the following set of rules:
# - In NetBuilder's syntax, blob's declaration and definition occur at the same time - when we define an operator which writes its output into a blob with a given name;
# - NetBuilder keeps track of all operator outputs seen before current execution point in the same block and up the stack in parent blocks.
# - If an operator writes into a previously unseen blob, it creates a **local** blob that is visible only within the current block and the subsequent children blocks. Local blobs created in a given block are effectively deleted when we exit the block. Any write into previously defined (in the same block or in the parent blocks) blob updates an originally created blob and does not result in the redefinition of a blob.
# - Operator's input blobs have to be defined earlier in the same block or in the parent blocks up the stack.
# As a result, in order to see values computed by a block after the block is finished the corresponding blobs have to be defined outside of the block. Note, that this is one of the ways to solve the problem with uninitialized blobs (e.g. blob created by 'then' branch, but not by 'else' branch), these rules effectively force visible blobs to be always correctly initialized.
def _pipe_step(input,
output=None,
num_threads=1,
processor=None,
name=None,
capacity=None,
group=None,
final_outputs=None):
"""
"""
if isinstance(input, Reader):
reader = input
elif hasattr(input, 'reader'):
reader = input.reader()
else:
raise ValueError('in must be a reader, queue or streaam.')
if processor is not None:
reader = ProcessingReader(reader, processor)
if num_threads == 0:
assert output is None
return reader, None
if name is None and processor is not None:
name = processor_name(processor)
if name is None and output is not None:
name = 'pipe_into:%s' % processor_name(output)
if name is None:
name = 'pipe_from:%s' % processor_name(input)
with Task(name=name, group=group, outputs=final_outputs) as task:
global_exit_net = core.Net('exit')
global_init_net = core.Net('init')
reader.setup_ex(global_init_net, global_exit_net)
out_queue = None
writer = None
steps = []
for thread_id in range(num_threads):
with NetBuilder(name='t:%d' % thread_id) as nb:
init_net = core.Net('init')
exit_net = core.Net('exit')
read_nets, status, rec = reader.read_record_ex(
init_net, exit_net)
if rec is not None:
if writer is None:
# hack so that the out queue gets the right name prefix
# (otherwise they would be prefixed with the thread id)
with NetBuilder(_fullname=task.name):
out_queue, writer = _init_output(
output, capacity, global_init_net,
global_exit_net)
write_nets, _ = writer.write_record_ex(
rec, init_net, exit_net, status)
else:
write_nets = []
ops.net(init_net)
ops.net(
core.execution_step('body',
list(read_nets) + list(write_nets),
should_stop_blob=status))
ops.net(exit_net)
steps.append(core.to_execution_step(nb))
ops.net(global_init_net)
ops.net(core.execution_step('body', steps, concurrent_substeps=True))
ops.net(global_exit_net)
return out_queue, task
def set_step(self, step):
self._assert_not_used()
self._step = core.to_execution_step(step)
def test_if_net(self):
with NetBuilder() as nb:
x0 = ops.Const(0)
x1 = ops.Const(1)
x2 = ops.Const(2)
y0 = ops.Const(0)
y1 = ops.Const(1)
y2 = ops.Const(2)
# basic logic
first_res = ops.Const(0)
with ops.IfNet(ops.Const(True)):
ops.Const(1, blob_out=first_res)
with ops.Else():
ops.Const(2, blob_out=first_res)
second_res = ops.Const(0)
with ops.IfNet(ops.Const(False)):
ops.Const(1, blob_out=second_res)
with ops.Else():
ops.Const(2, blob_out=second_res)
# nested and sequential ifs,
# empty then/else,
# passing outer blobs into branches,
# writing into outer blobs, incl. into input blob
# using local blobs
with ops.IfNet(ops.LT([x0, x1])):
local_blob = ops.Const(900)
ops.Add([ops.Const(100), local_blob], [y0])
gt = ops.GT([x1, x2])
with ops.IfNet(gt):
# empty then
pass
with ops.Else():
ops.Add([y1, local_blob], [local_blob])
ops.Add([ops.Const(100), y1], [y1])
with ops.IfNet(ops.EQ([local_blob, ops.Const(901)])):
ops.Const(7, blob_out=y2)
ops.Add([y1, y2], [y2])
with ops.Else():
# empty else
pass
plan = Plan('if_net_test')
plan.AddStep(to_execution_step(nb))
ws = workspace.C.Workspace()
ws.run(plan)
first_res_value = ws.blobs[str(first_res)].fetch()
second_res_value = ws.blobs[str(second_res)].fetch()
y0_value = ws.blobs[str(y0)].fetch()
y1_value = ws.blobs[str(y1)].fetch()
y2_value = ws.blobs[str(y2)].fetch()
self.assertEquals(first_res_value, 1)
self.assertEquals(second_res_value, 2)
self.assertEquals(y0_value, 1000)
self.assertEquals(y1_value, 101)
self.assertEquals(y2_value, 108)
self.assertTrue(str(local_blob) not in ws.blobs)
def _static_threads_task(name, group, final_outputs, reader, num_threads,
output, capacity):
node_name = str(Node.current())
profiler_name = "{0}/{1}/{2}/{3}/{4}".format(
node_name,
"pipe",
name,
processor_name(input) if input else "NoInput",
processor_name(output) if output else "NoOutput")
with Task(name=name, group=group, outputs=final_outputs) as task:
global_exit_net = core.Net('exit')
global_init_net = core.Net('init')
reader.setup_ex(global_init_net, global_exit_net)
out_queue = None
writer = None
steps = []
for thread_id in range(num_threads):
with NetBuilder(name='t:%d' % thread_id) as nb:
init_net = core.Net('init')
exit_net = core.Net('exit')
read_nets, status, rec = reader.read_record_ex(
init_net, exit_net)
init_net.ConstantFill(
[], [status],
shape=[],
value=False,
dtype=core.DataType.BOOL
)
if rec is not None:
if writer is None:
# hack so that the out queue gets the right name prefix
# (otherwise they would be prefixed with the thread id)
with NetBuilder(_fullname=task.name):
out_queue, writer = _init_output(
output, capacity, global_init_net,
global_exit_net)
write_nets, _ = writer.write_record_ex(
rec, init_net, exit_net, status)
else:
write_nets = []
timer_start_net = core.Net('timer_start')
timer = timer_start_net.TimerBegin([], counter_name=profiler_name)
timer_end_net = core.Net('timer_end')
timer_end_net.TimerEnd(timer, [])
ops.net(init_net)
ops.net(core.execution_step(
'body',
[timer_start_net] + list(read_nets) + list(write_nets) +
[timer_end_net],
should_stop_blob=status))
ops.net(timer_end_net)
ops.net(exit_net)
steps.append(core.to_execution_step(nb))
ops.net(global_init_net)
ops.net(core.execution_step('body', steps, concurrent_substeps=True))
ops.net(global_exit_net)
return out_queue, task
def setup(self, net):
if self.type == _SetupBuilder.INIT:
return core.to_execution_step(self)
def setup(self, net):
if self.type == _SetupBuilder.INIT:
return core.to_execution_step(self)
def exit(self, net):
if self.type == _SetupBuilder.EXIT:
return core.to_execution_step(self)
def exit(self, net):
if self.type == _SetupBuilder.EXIT:
return core.to_execution_step(self)
def _pipe_step(input,
output=None,
num_threads=1,
processor=None,
name=None,
capacity=None,
group=None,
final_outputs=None):
"""
"""
if isinstance(input, Reader):
reader = input
elif hasattr(input, 'reader'):
reader = input.reader()
else:
raise ValueError('in must be a reader, queue or streaam.')
if processor is not None:
reader = ProcessingReader(reader, processor)
if num_threads == 0:
assert output is None
return reader, None
if name is None and processor is not None:
name = processor_name(processor)
if name is None and output is not None:
name = 'pipe_into:%s' % processor_name(output)
if name is None:
name = 'pipe_from:%s' % processor_name(input)
node_name = str(Node.current())
profiler_name = "{0}/{1}/{2}/{3}/{4}".format(
node_name, "pipe", name,
processor_name(input) if input else "NoInput",
processor_name(output) if output else "NoOutput")
with Task(name=name, group=group, outputs=final_outputs) as task:
global_exit_net = core.Net('exit')
global_init_net = core.Net('init')
reader.setup_ex(global_init_net, global_exit_net)
out_queue = None
writer = None
steps = []
for thread_id in range(num_threads):
with NetBuilder(name='t:%d' % thread_id) as nb:
init_net = core.Net('init')
exit_net = core.Net('exit')
read_nets, status, rec = reader.read_record_ex(
init_net, exit_net)
init_net.ConstantFill([], [status],
shape=[],
value=False,
dtype=core.DataType.BOOL)
if rec is not None:
if writer is None:
# hack so that the out queue gets the right name prefix
# (otherwise they would be prefixed with the thread id)
with NetBuilder(_fullname=task.name):
out_queue, writer = _init_output(
output, capacity, global_init_net,
global_exit_net)
write_nets, _ = writer.write_record_ex(
rec, init_net, exit_net, status)
else:
write_nets = []
timer_start_net = core.Net('timer_start')
timer = timer_start_net.TimerBegin([],
counter_name=profiler_name)
timer_end_net = core.Net('timer_end')
timer_end_net.TimerEnd(timer, [])
ops.net(init_net)
ops.net(
core.execution_step('body',
[timer_start_net] + list(read_nets) +
list(write_nets) + [timer_end_net],
should_stop_blob=status))
ops.net(timer_end_net)
ops.net(exit_net)
steps.append(core.to_execution_step(nb))
ops.net(global_init_net)
ops.net(core.execution_step('body', steps, concurrent_substeps=True))
ops.net(global_exit_net)
return out_queue, task
ops.Copy("one", "y")
with ops.Else():
ops.Copy("zero", "y")
# Note the usage of `NetBuilder`'s `ops.IfNet` and `ops.Else` calls: `ops.IfNet` accepts a blob reference or blob name as an input, it expects an input blob to have a scalar value convertible to bool. Note that the optional `ops.Else` is at the same level as `ops.IfNet` and immediately follows the corresponding `ops.IfNet`. Let's execute the resulting net (execution step) and check the values of the blobs.
#
# Note that since x = 0.5, which is indeed greater than 0, we should expect y = 1 after execution.
# In[3]:
# Initialize a Plan
plan = Plan('if_net_test')
# Add the NetBuilder definition above to the Plan
plan.AddStep(to_execution_step(nb))
# Initialize workspace for blobs
ws = workspace.C.Workspace()
# Run the Plan
ws.run(plan)
# Fetch some blobs and print
print('x = ', ws.blobs["x"].fetch())
print('y = ', ws.blobs["y"].fetch())
# Before going further, it's important to understand the semantics of execution blocks ('then' and 'else' branches in the example above), i.e. handling of reads and writes into global (defined outside of the block) and local (defined inside the block) blobs.
#
# `NetBuilder` uses the following set of rules:
#
# - In `NetBuilder`'s syntax, a blob's declaration and definition occur at the same time - we define an operator which writes its output into a blob with a given name.
#
def set_step(self, step):
self._assert_not_used()
self._step = core.to_execution_step(step)
def test_if_net(self):
with NetBuilder() as nb:
x0 = ops.Const(0)
x1 = ops.Const(1)
x2 = ops.Const(2)
y0 = ops.Const(0)
y1 = ops.Const(1)
y2 = ops.Const(2)
# basic logic
first_res = ops.Const(0)
with ops.IfNet(ops.Const(True)):
ops.Const(1, blob_out=first_res)
with ops.Else():
ops.Const(2, blob_out=first_res)
second_res = ops.Const(0)
with ops.IfNet(ops.Const(False)):
ops.Const(1, blob_out=second_res)
with ops.Else():
ops.Const(2, blob_out=second_res)
# nested and sequential ifs,
# empty then/else,
# passing outer blobs into branches,
# writing into outer blobs, incl. into input blob
# using local blobs
with ops.IfNet(ops.LT([x0, x1])):
local_blob = ops.Const(900)
ops.Add([ops.Const(100), local_blob], [y0])
gt = ops.GT([x1, x2])
with ops.IfNet(gt):
# empty then
pass
with ops.Else():
ops.Add([y1, local_blob], [local_blob])
ops.Add([ops.Const(100), y1], [y1])
with ops.IfNet(ops.EQ([local_blob, ops.Const(901)])):
ops.Const(7, blob_out=y2)
ops.Add([y1, y2], [y2])
with ops.Else():
# empty else
pass
plan = Plan('if_net_test')
plan.AddStep(to_execution_step(nb))
ws = workspace.C.Workspace()
ws.run(plan)
first_res_value = ws.blobs[str(first_res)].fetch()
second_res_value = ws.blobs[str(second_res)].fetch()
y0_value = ws.blobs[str(y0)].fetch()
y1_value = ws.blobs[str(y1)].fetch()
y2_value = ws.blobs[str(y2)].fetch()
self.assertEquals(first_res_value, 1)
self.assertEquals(second_res_value, 2)
self.assertEquals(y0_value, 1000)
self.assertEquals(y1_value, 101)
self.assertEquals(y2_value, 108)
self.assertTrue(str(local_blob) not in ws.blobs)
