def testAssignOnWorkerLost(self):
r"""
Proper initial allocation should divide the graph like
0FL 1FL 2F 3F 4R 5R | 6FL 7FL 8F 9F 10R 11R
| x | | x | | x | | | x | | x | | x |
12R 13R 14R 15R 16U 17U | 18R 19R 20R 21R 22U 23U
U: UNSCHEDULED F: FINISHED R: READY L: LOST
"""
op_states = dict()
graph, str_to_chunk = self._build_chunk_dag(
'0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, '
'19, 20, 21, 22, 23',
'0 -> 12, 0 -> 13, 1 -> 12, 1 -> 13, 2 -> 14, 2 -> 15, 3 -> 14, 3 -> 15, '
'4 -> 16, 4 -> 17, 5 -> 16, 5 -> 17, 6 -> 18, 6 -> 19, 7 -> 18, 7 -> 19, '
'8 -> 20, 8 -> 21, 9 -> 20, 9 -> 21, 10 -> 22, 10 -> 23, 11 -> 22, 11 -> 23'
)
inputs = [(str_to_chunk[str(i)], str_to_chunk[str(i + 1)])
for i in range(0, 12, 2)]
results = [(str_to_chunk[str(i)], str_to_chunk[str(i + 1)])
for i in range(12, 24, 2)]
fixed_assigns = dict()
for idx in range(4):
for i in range(2):
fixed_assigns[inputs[idx][i].op.key] = f'w{idx % 2 + 1}'
op_states[inputs[idx][i].op.key] = OperandState.FINISHED
fixed_assigns[results[idx][i].op.key] = f'w{idx % 2 + 1}'
op_states[results[idx][i].op.key] = OperandState.READY
for inp in inputs:
for n in inp:
if n.op.key in fixed_assigns:
continue
op_states[n.op.key] = OperandState.READY
lost_chunks = [c.key for inp in (inputs[0], inputs[2]) for c in inp]
worker_metrics = dict(w2=24, w3=24)
analyzer = GraphAnalyzer(graph, worker_metrics, fixed_assigns,
op_states, lost_chunks)
changed_states = analyzer.analyze_state_changes()
self.assertEqual(len(changed_states), 8)
self.assertTrue(
all(changed_states[c.op.key] == OperandState.READY
for inp in (inputs[0], inputs[2]) for c in inp))
self.assertTrue(
all(changed_states[c.op.key] == OperandState.UNSCHEDULED
for res in (results[0], results[2]) for c in res))
assignments = analyzer.calc_operand_assignments(
analyzer.get_initial_operand_keys())
for inp in inputs:
if any(n.op.key in fixed_assigns for n in inp):
continue
self.assertEqual(1, len(set(assignments[n.op.key] for n in inp)))
worker_assigns = dict((k, 0) for k in worker_metrics)
for w in assignments.values():
worker_assigns[w] += 1
self.assertEqual(2, worker_assigns['w2'])
self.assertEqual(6, worker_assigns['w3'])
def testAssignOnWorkerLost(self):
import numpy as np
from mars.scheduler import OperandState
from mars.tensor.random import TensorRandint
from mars.tensor.arithmetic import TensorTreeAdd
graph = DAG()
r"""
Proper initial allocation should divide the graph like
FL FL F F R R | FL FL F F R R
| x | | x | | x | | | x | | x | | x |
R R R R U U | R R R R U U
U: UNSCHEDULED F: FINISHED R: READY L: LOST
"""
op_states = dict()
inputs = [
tuple(
TensorRandint(
dtype=np.float32()).new_chunk(None, shape=(10, 10))
for _ in range(2)) for _ in range(6)
]
results = [
tuple(
TensorTreeAdd(_key=f'{i}_{j}', dtype=np.float32()).new_chunk(
None, shape=(10, 10)) for j in range(2)) for i in range(6)
]
for inp, outp in zip(inputs, results):
for o in outp:
o.op._inputs = list(inp)
op_states[o.op.key] = OperandState.UNSCHEDULED
graph.add_node(o)
for n in inp:
op_states[n.op.key] = OperandState.UNSCHEDULED
graph.add_node(n)
for o in outp:
graph.add_edge(n, o)
fixed_assigns = dict()
for idx in range(4):
for i in range(2):
fixed_assigns[inputs[idx][i].op.key] = f'w{idx % 2 + 1}'
op_states[inputs[idx][i].op.key] = OperandState.FINISHED
fixed_assigns[results[idx][i].op.key] = f'w{idx % 2 + 1}'
op_states[results[idx][i].op.key] = OperandState.READY
for inp in inputs:
for n in inp:
if n.op.key in fixed_assigns:
continue
op_states[n.op.key] = OperandState.READY
lost_chunks = [c.key for inp in (inputs[0], inputs[2]) for c in inp]
worker_metrics = dict(w2=24, w3=24)
analyzer = GraphAnalyzer(graph, worker_metrics, fixed_assigns,
op_states, lost_chunks)
changed_states = analyzer.analyze_state_changes()
self.assertEqual(len(changed_states), 8)
self.assertTrue(
all(changed_states[c.op.key] == OperandState.READY
for inp in (inputs[0], inputs[2]) for c in inp))
self.assertTrue(
all(changed_states[c.op.key] == OperandState.UNSCHEDULED
for res in (results[0], results[2]) for c in res))
assignments = analyzer.calc_operand_assignments(
analyzer.get_initial_operand_keys())
for inp in inputs:
if any(n.op.key in fixed_assigns for n in inp):
continue
self.assertEqual(1, len(set(assignments[n.op.key] for n in inp)))
worker_assigns = dict((k, 0) for k in worker_metrics)
for w in assignments.values():
worker_assigns[w] += 1
self.assertEqual(2, worker_assigns['w2'])
self.assertEqual(6, worker_assigns['w3'])