def test_3p_get_common_blocks(self):
dp_ids = [1, 2, 3]
dp_block_sizes = {
1: {
'1': 100
},
2: {
'1': 100,
'2': 100
},
3: {
'1': 100,
'2': 100
}
}
common_blocks = _get_common_blocks(dp_block_sizes, dp_ids)
assert '1' in common_blocks
assert len(common_blocks['1']) == 3
block_1_set = set(common_blocks['1'])
# Should have (1, 2), (1, 3), (2, 3)
for dpcombo in [(1, 2), (1, 3), (2, 3)]:
assert dpcombo in block_1_set
assert '2' in common_blocks
assert len(common_blocks['2']) == 1
assert common_blocks['2'][0] == (2, 3)
def test_basic_3p(self):
dp_ids = [1, 2, 3]
dp_block_sizes = {
1: {
'1': 100
},
2: {
'1': 100,
'2': 100
},
3: {
'1': 100,
'2': 100
},
}
block_lookups = {1: {'1': 1}, 2: {'1': 1, '2': 2}, 3: {'1': 1, '2': 2}}
blocks = _get_common_blocks(dp_block_sizes, dp_ids)
chunks = _create_work_packages(blocks,
dp_block_sizes,
dp_ids,
log,
block_lookups,
chunk_size_aim=100)
# Case I: all blocks need to be chunked
# Block 1 should create 100 chunks between dp combinations: 1:2, 1:3, and 2:3 for 300 chunks
# Block 2 should create 100 chunks between 2:3
assert len(chunks) == 300 + 100
# Case II: each block fits into exactly one work package
blocks = _get_common_blocks(dp_block_sizes, dp_ids)
chunks = _create_work_packages(blocks,
dp_block_sizes,
dp_ids,
log,
block_lookups,
chunk_size_aim=10000)
assert len(chunks) == 4
# Case III: all blocks fit into one work package
blocks = _get_common_blocks(dp_block_sizes, dp_ids)
chunks = _create_work_packages(blocks,
dp_block_sizes,
dp_ids,
log,
block_lookups,
chunk_size_aim=40000)
assert len(chunks) == 1
def test_basic_3p(self):
dp_ids = [1, 2, 3]
dp_block_sizes = {
1: {
'1': 100
},
2: {
'1': 100,
'2': 100
},
3: {
'1': 100,
'2': 100
},
}
blocks = _get_common_blocks(dp_block_sizes, dp_ids)
chunks = _create_work_chunks(blocks, dp_block_sizes, dp_ids, log, 100)
# Block 1 should create 100 chunks between dp combinations: 1:2, 1:3, and 2:3 for 300 chunks
# Block 2 should create 100 chunks between 2:3
assert len(chunks) == 300 + 100
def test_2p_single_chunked_block(self):
dp_ids = [1, 2]
dp_block_sizes = {1: {'1': 100}, 2: {'1': 100, '2': 100}}
blocks = _get_common_blocks(dp_block_sizes, dp_ids)
block_lookups = {1: {'1': 1}, 2: {'1': 1, '2': 2}}
chunks = _create_work_packages(blocks,
dp_block_sizes,
dp_ids,
log,
block_lookups,
chunk_size_aim=100)
assert len(chunks) == 100
for package in chunks:
for chunk_pair in package:
for c in chunk_pair:
assert "range" in c
lower, upper = c['range']
assert lower < upper
assert upper - lower <= 10
assert "block_id" in c
assert "datasetIndex" in c
assert "dataproviderId" in c
def test_3p_get_common_blocks(self):
dp_ids = [1, 2, 3]
dp_block_sizes = {
1: {
'1': 100
},
2: {
'1': 100,
'2': 100
},
3: {
'1': 100,
'2': 100
}
}
common_blocks = set(_get_common_blocks(dp_block_sizes, dp_ids))
assert ('1', (1, 2)) in common_blocks
assert ('1', (1, 3)) in common_blocks
assert ('1', (2, 3)) in common_blocks
assert ('2', (2, 3)) in common_blocks
assert len(common_blocks) == 4
def test_2p_get_common_blocks(self):
dp_ids = [33, 34]
dp_block_sizes = {33: {'1': 100}, 34: {'1': 100, '2': 100}}
common_blocks = _get_common_blocks(dp_block_sizes, dp_ids)
assert list(common_blocks) == [("1", (33, 34))]
def test_2p_get_common_blocks(self):
dp_ids = [33, 34]
dp_block_sizes = {33: {'1': 100}, 34: {'1': 100, '2': 100}}
common_blocks = _get_common_blocks(dp_block_sizes, dp_ids)
assert common_blocks == {"1": [(33, 34)]}