def test_cycle_detection():
#1. simple linear graph with cycle
b = IdentityProcessor()
c = IdentityProcessor()(b)
d = IdentityProcessor()(c)
b(c)
assert has_cycle([b]), '#1 Cycle not detected'
#2. More complex non linear graph
a1 = IntProducer()
b1 = IdentityProcessor()(a1)
c1 = IdentityProcessor()(b1)
d1 = IdentityProcessor()(a1)
e1 = IdentityProcessor()
f1 = JoinerProcessor()(e1, d1)
g1 = JoinerProcessor()(c1, b1, d1)
e1(g1)
assert not has_cycle([e1]), "#2 Cycle detected"
assert not has_cycle([a1]), "#3 Cycle not detected"
a2 = IntProducer()
b2 = IdentityProcessor()(a2)
c2 = IdentityProcessor()(b2)
d2 = IdentityProcessor()(a2)
e2 = IdentityProcessor()
f2 = JoinerProcessor()(e2, d2)
g2 = JoinerProcessor()(c2, b2, f2)
e2(g2)
assert has_cycle([e2]), '#4 Cycle not detected'
assert has_cycle([a2]), "#5 Cycle not detected"
def test_nb_tasks_created():
#1. Test that the number of tasks created is equal to number of nodes
A = IntProducer()
B = IdentityProcessor()(A)
C = IdentityProcessor()(B)
D = JoinerProcessor()(C)
E = JoinerProcessor()(B, C, D)
tsort = topological_sort([A])
tasks_data = _task_data_from_node_tsort(tsort)
ee = RealtimeExecutionEngine()
ee._al_create_processes(tasks_data)
assert len(tsort) == len(ee._procs)
def test_mp_example1():
producer = IntProducer(0, 40, 0.1)
identity = IdentityProcessor(nb_tasks = 5)(producer)
identity1 = IdentityProcessor(nb_tasks = 5)(identity)
joined = JoinerProcessor(nb_tasks = 5)(identity, identity1)
printer = CommandlineConsumer()(joined)
flow = Flow([producer], [printer])
flow.run()
flow.join()
def test_taskmodulenode_example1():
producer = IntProducer(0, 40, 0.05)
identity = IdentityProcessor(nb_tasks = 1)(producer)
identity1 = IdentityProcessor(nb_tasks = 1)(identity)
joined = JoinerProcessor(nb_tasks = 1)(identity, identity1)
task_module = TaskModuleNode(identity, joined)
printer = CommandlineConsumer()(task_module)
flow = Flow([producer], [printer])
flow.run()
flow.join()
def test_nb_tasks_created_1():
#2. Test that number of tasks created is different than number of
# nodes, in the case of TaskModuleProcessor
zero = IntProducer()
a = IdentityProcessor()(zero)
b = IdentityProcessor()(a)
c = IdentityProcessor()(b)
d = JoinerProcessor()(b, c)
e = IdentityProcessor()(d)
f = JoinerProcessor()(d, e, c, b)
module = TaskModuleNode(a, f)
out = CommandlineConsumer()(module)
tsort = topological_sort([zero])
tasks_data = _task_data_from_node_tsort(tsort)
ee = RealtimeExecutionEngine()
ee._al_create_processes(tasks_data)
assert len(ee._procs) == 3
def test_graph_with_no_consumer():
# Graph with no consumer should run.
producer = IntProducer(0, 40, 0.05)
identity = IdentityProcessor(nb_tasks=1)(producer)
identity1 = IdentityProcessor(nb_tasks=1)(identity)
joined = JoinerProcessor(nb_tasks=1)(identity, identity1)
task_module = TaskModuleNode(identity, joined)
flow = Flow([producer], [])
flow.run()
flow.join()
def test_taskmodule_node_3():
'''
Test error when trying to put module inside of moduel
'''
#2. Tests module inside of module
zero = IntProducer()
a = IdentityProcessor()(zero)
b = IdentityProcessor()(a)
c = IdentityProcessor()(b)
d = JoinerProcessor()(b, c)
e = IdentityProcessor()(d)
f = JoinerProcessor()(d, e, c, b)
module = TaskModuleNode(a, f)
g = IdentityProcessor()(module)
h = IdentityProcessor()(g)
i = IdentityProcessor()(h)
with pytest.raises(ValueError):
module1 = TaskModuleNode(module, i)
def test_graph_with_deadend_processor():
# Graph with no consumer should run.
producer = IntProducer(0, 40, 0.05)
identity = IdentityProcessor(nb_tasks=1)(producer)
identity1 = IdentityProcessor(nb_tasks=1)(identity)
joined = JoinerProcessor(nb_tasks=1)(identity, identity1)
task_module = TaskModuleNode(identity, joined)
dead_end = IdentityProcessor()(task_module)
printer = CommandlineConsumer()(task_module)
flow = Flow([producer], [printer])
flow.run()
flow.join()
def test_gpu_nodes_not_accepted(monkeypatch):
#2. Test that gpu node rejects because already all gpus were allocated
# to other nodes.
def gpus_mock():
return [0, 1]
monkeypatch.setattr(videoflow.utils.system,
'get_gpus_available_to_process', gpus_mock)
A = IntProducer()
B = IdentityProcessorGpuOnly()(A)
C = IdentityProcessorGpuOnly()(B)
D = IdentityProcessorGpuOnly()(C)
E = JoinerProcessor()(D)
F = JoinerProcessor()(B, C, D, E)
tsort = topological_sort([A])
tasks_data = _task_data_from_node_tsort(tsort)
ee = RealtimeExecutionEngine()
with pytest.raises(RuntimeError):
ee._al_create_processes(tasks_data)
def test_taskmodule_node():
'''
Tests simple task module creation
and tests that it can be part of a flow
'''
#1. Tests simple module first
zero = IntProducer()
a = IdentityProcessor()(zero)
b = IdentityProcessor()(a)
c = IdentityProcessor()(b)
d = JoinerProcessor()(b, c)
e = IdentityProcessor()(d)
f = JoinerProcessor()(d, e, c, b)
module = TaskModuleNode(a, f)
out = CommandlineConsumer()(module)
#2. Tests that you raise an exception as error here.
with pytest.raises(RuntimeError):
out1 = CommandlineConsumer()(f)
graph_engine = GraphEngine([zero], [out])
tsort = graph_engine.topological_sort()
assert len(tsort) == 3
from videoflow.core import Flow
from videoflow.producers import IntProducer
from videoflow.processors import IdentityProcessor, JoinerProcessor
from videoflow.consumers import CommandlineConsumer
producer = IntProducer(0, 40, 0.1)
identity = IdentityProcessor()(producer)
identity1 = IdentityProcessor()(identity)
joined = JoinerProcessor()(identity, identity1)
printer = CommandlineConsumer()(joined)
flow = Flow([producer], [printer],maintain_states=True,state_config={"flow_name":"simple_example","save_interval":25})
flow.run()
flow.join()
from videoflow.core import Flow from videoflow.producers import IntProducer from videoflow.processors import IdentityProcessor, JoinerProcessor from videoflow.consumers import CommandlineConsumer producer = IntProducer(0, 40, 0.1) identity = IdentityProcessor(nb_tasks=5)(producer) identity1 = IdentityProcessor(nb_tasks=5)(identity) joined = JoinerProcessor(nb_tasks=5)(identity, identity1) printer = CommandlineConsumer()(joined) flow = Flow([producer], [printer]) flow.run() flow.join()
from videoflow.core import Flow
from videoflow.producers import IntProducer
from videoflow.processors import IdentityProcessor, JoinerProcessor
from videoflow.consumers import CommandlineConsumer
from videoflow.core.constants import BATCH
reader = IntProducer(0, 100, 0.001)
game_state_processor = IdentityProcessor(fps=6, nb_tasks=1, name='i1')(reader)
hero_processors = JoinerProcessor()(reader, game_state_processor)
ability_processor = JoinerProcessor()(reader, game_state_processor,
hero_processors)
ammo_processor = JoinerProcessor()(reader, game_state_processor)
death_processor = JoinerProcessor()(reader, game_state_processor)
hp_processor = JoinerProcessor()(reader, game_state_processor)
killfeed_processor = JoinerProcessor(fps=1, nb_tasks=5)(reader,
game_state_processor)
map_processor = JoinerProcessor()(reader, game_state_processor)
resurrect_processor = JoinerProcessor()(reader, game_state_processor)
sr_processor = JoinerProcessor()(reader, game_state_processor)
ultimate_processor = JoinerProcessor()(reader, game_state_processor)
player_score_processor = JoinerProcessor()(reader, game_state_processor)
consumer_before = JoinerProcessor()(
reader, game_state_processor, hero_processors, death_processor,
killfeed_processor, ammo_processor, hp_processor, ultimate_processor,
ability_processor, player_score_processor, map_processor, sr_processor,
resurrect_processor)
consumer = CommandlineConsumer()(consumer_before)
flow = Flow([reader], [consumer], flow_type=BATCH)
flow.run()
flow.join()