def test_replace_block_id(self):
bid1 = BlockID('d1g0006300000001230101c62d8736c72800020000000001')
bid2 = BlockID('d1g0006300000001230101c62d8736c72800020000000002')
bid3 = BlockID('d1g0006300000001230101c62d8736c72800020000000003')
bid4 = BlockID('d1g0006300000001230101c62d8736c72800020000000004')
bid5 = BlockID('d1g0006300000001230101c62d8736c72800020000000005')
bid6 = BlockID('d1g0006300000001230101c62d8736c72800020000000006')
region_levels = [
[['aa', 'ee', {'block_id': bid1}], ['hh', 'zz', {'block_id': bid2}]],
[['ea', 'ff', {'block_id': bid4}], ['mm', 'yy', {'block_id': bid5}]],
]
excepted_region_levels = [
[['aa', 'ee', BlockDesc({'block_id': bid1})], ['hh', 'zz', BlockDesc({'block_id': bid2})]],
[['ea', 'ff', BlockDesc({'block_id': bid3})], ['mm', 'yy', BlockDesc({'block_id': bid5})]],
]
region = Region(levels=region_levels)
region.replace_block_id(bid4, bid3)
self.assertEqual(excepted_region_levels, region['levels'])
self.assertRaises(BlockNotInRegion, region.replace_block_id, bid6, bid1)
def test_find_merge(self):
region_levels_cases = [
[[[
['aa', 'ee', {'size': 8}],
['ee', 'ff', {'size': 16}],
['pp', 'zz', {'size': 8}],
['zz', None, {'size': 4}]
],
[
['aa', 'pz', {'size': 4}],
['qq', 'zz', {'size': 8}]
]],
(1, ['qq', 'zz', BlockDesc(size=8)], [['pp', 'zz', BlockDesc(size=8)]])],
[[[
['aa', 'ee', {'size': 8}],
['ee', 'ff', {'size': 8}],
['hh', 'hz', {'size': 8}]
],
[
['mm', 'yy', {'size': 8}]
]],
None]
]
for levels, excepted in region_levels_cases:
region = Region(levels=levels)
res = region.find_merge()
self.assertEqual(excepted, res)
def test_json(self):
blk = BlockDesc({
'block_id':
BlockID('d0', 'g000640000000123', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1),
'size':
1000,
'range': ['0a', '0b'],
'ts_range': ["1235", "456"],
'ref_num':
1,
'is_del':
0,
'mtime':
1,
})
rst = utfjson.dump(blk)
expected = (
'{"block_id": "d0g0006400000001230000idc000c62d8736c72800020000000001", "is_del": 0, "ref_num": 1, "range": ["0a", "0b"], "mtime": 1, "ts_range": ["1235", "456"], "size": 1000}'
)
self.assertEqual(expected, rst)
loaded = BlockDesc(utfjson.load(rst))
self.assertEqual(blk, loaded)
def test_json(self):
region = Region({
'range': ['a', 'z'],
'levels':
[[['a', 'b', BlockDesc()],
[
'b', 'c',
BlockDesc(
size=2,
block_id=BlockID(
'd1g0006300000001230101idc000c62d8736c72800020000000001'
))
]]]
})
rst = utfjson.dump(region)
expected = (
'{"range": ["a", "z"], "levels": [[["a", "b", '
'{"is_del": 0, "range": null, "block_id": null, "size": 0}], '
'["b", "c", {"is_del": 0, "range": null, '
'"block_id": "d1g0006300000001230101idc000c62d8736c72800020000000001", "size": 2}]]], "idc": ""}'
)
self.assertEqual(expected, rst)
loaded = Region(utfjson.load(rst))
self.assertEqual(region, loaded)
def test_blockdesc(self):
block_id = 'd1g0006300000001230101idc000c62d8736c72800020000000001'
cases = ((None, {
'block_id': None,
'size': 0,
'range': None,
'ts_range': None,
'ref_num': 0,
'is_del': 0
}), ({
'block_id': block_id,
'range': ['a', 'b'],
'ts_range': ["124", None],
'ref_num': 0,
'size': 34,
'mtime': 1,
'is_del': 0
}, {
'block_id': BlockID(block_id),
'range': rangeset.Range('a', 'b'),
'ts_range': ["124", None],
'ref_num': 0,
'size': 34,
'mtime': 1,
'is_del': 0
}), ({
'block_id': BlockID(block_id),
'range': rangeset.Range('b', 'bb'),
'ts_range': ["1235", "456"],
'ref_num': 0,
'mtime': 1
}, {
'block_id': BlockID(block_id),
'range': rangeset.Range('b', 'bb'),
'ts_range': ["1235", "456"],
'ref_num': 0,
'size': 0,
'mtime': 1,
'is_del': 0,
}))
for b, expected in cases:
if b is None:
blk = BlockDesc()
expected['mtime'] = blk["mtime"]
else:
blk = BlockDesc(b)
self.assertEqual(expected, blk)
self.assertRaises(ValueError, BlockDesc, is_del='a')
self.assertRaises(ValueError, BlockDesc, size='a')
self.assertRaises(KeyError, BlockDesc, a=3)
def test_init(self):
region_cases = [
(
{},
{'idc': '', 'range': None, 'levels': []}
),
(
{'range': ['a', 'b'], 'idc': '.bei'},
{'idc': '.bei', 'range': ['a', 'b'], 'levels': []}
),
(
{'levels': [[['a', 'b', BlockDesc()]], [['c', 'd', BlockDesc(size=1)]]]},
{'idc': '', 'range': None, 'levels': [[['a', 'b', BlockDesc()]], [['c', 'd', BlockDesc(size=1)]]]}
),
(
{'range': ['a', 'z'], 'levels': [[['a', 'b', BlockDesc()], ['b', 'c', BlockDesc(size=2)]]]},
{'idc': '', 'range': ['a', 'z'], 'levels': [[['a', 'b', BlockDesc()], ['b', 'c', BlockDesc(size=2)]]]}
),
]
for case, excepted in region_cases:
region = Region(case)
self.assertEqual(excepted, region)
region_cases_argkv = [
(
[[['a', 'b', BlockDesc(size=1)], ['c', 'd', BlockDesc(size=2)]]],
{'idc': '', 'range': None, 'levels': [[['a', 'b', BlockDesc(size=1)], ['c', 'd', BlockDesc(size=2)]]]}
),
(
['a', 'z'],
{'idc': '', 'range': ['a', 'z'], 'levels': []}
),
(
[],
{'idc': '', 'range': ['a', 'z'], 'levels': [
[['a', 'b', BlockDesc(size=1)], ['c', 'd', BlockDesc(size=2)]]]}
),
]
region = Region(levels=region_cases_argkv[0][0])
self.assertEqual(region_cases_argkv[0][1], region)
region = Region(range=region_cases_argkv[1][0])
self.assertEqual(region_cases_argkv[1][1], region)
region = Region(levels=region_cases_argkv[0][0], range=region_cases_argkv[1][0])
self.assertEqual(region_cases_argkv[2][1], region)
def test_json(self):
blk = BlockDesc({
'block_id': BlockID('d0', 'g000640000000123', '0000',
DriveID('c62d8736c7280002'), 1),
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
rst = utfjson.dump(blk)
expected = ('{"is_del": 0, "range": ["0a", "0b"], "block_id": '
'"d0g0006400000001230000c62d8736c72800020000000001", "size": 1000}')
self.assertEqual(expected, rst)
loaded = BlockDesc(utfjson.load(rst))
self.assertEqual(blk, loaded)
def setUp(self):
self.foo_block = BlockDesc({
'block_id':
BlockID('d0', 'g000640000000123', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1),
'size':
1000,
'range': ['0a', '0b'],
'ts_range':
None,
'ref_num':
1,
'is_del':
0
})
def make_test_block_group(self, blk_idxes, config=None):
gid = 'g000640000000123'
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
if config is None:
config = _ec_config
num_idcs = sum(config['cross_idc'])
idcs = ['idc' + (str(i).rjust(3, '0')) for i in range(num_idcs)]
bg = BlockGroup(block_group_id=gid, idcs=idcs, config=config)
for i, bi in enumerate(blk_idxes):
bi = BlockIndex(bi)
typ = bg.get_block_type(bi)
drive_id = DriveID(idcs[int(bi[0])] + 'c62d8736c7280002')
blkid = BlockID(typ, gid, bi, drive_id, i)
blk = copy.deepcopy(base_blk)
blk['block_id'] = blkid
bg.add_block(blk)
return bg
def test_classify_blocks(self):
gid = 'g000640000000123'
g = BlockGroup(block_group_id=gid,
idcs=['a', 'b', 'c'],
config=_ec_config)
blks = g.classify_blocks(0, only_primary=True)
self.assertEqual([], blks['ec'] + blks['replica'] + blks['mark_del'])
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
ec_blk_idxes = ['0000', '0001']
replica_blk_idxes = ['0002', '0008', '0012']
mark_del_idxes = ['0003', '0004']
for i, idx in enumerate(ec_blk_idxes + replica_blk_idxes +
mark_del_idxes):
typ = g.get_block_type(idx)
blkid = BlockID(typ, gid, idx, DriveID('idc000'
'c62d8736c7280002'), i)
blk = copy.deepcopy(base_blk)
blk['block_id'] = blkid
if idx in mark_del_idxes:
blk['is_del'] = 1
g.add_block(blk)
for only_primary in (True, False):
blks = g.classify_blocks(0, only_primary)
blk_idxes = []
for blk in blks['ec'] + blks['replica'] + blks['mark_del']:
idx = BlockID(blk['block_id']).block_index
blk_idxes.append(idx)
expect_ids = copy.deepcopy(ec_blk_idxes)
#'0004' in ec_blk_idxes is parity, so should not in mark_del
if only_primary is True:
expect_ids += replica_blk_idxes[:1] + mark_del_idxes[:1]
else:
expect_ids += replica_blk_idxes + mark_del_idxes[:1]
self.assertEqual(expect_ids, blk_idxes)
def test_blockdesc(self):
block_id = 'd1g0006300000001230101c62d8736c72800020000000001'
cases = ((None,
{'block_id': None,
'size': 0,
'range': None,
'is_del': 0}),
({'block_id': block_id,
'range': ['a', 'b'],
'size': 34,
'is_del': 0},
{'block_id': BlockID(block_id),
'range': rangeset.Range('a', 'b'),
'size': 34,
'is_del': 0}),
({'block_id': BlockID(block_id),
'range': rangeset.Range('b', 'bb')},
{'block_id': BlockID(block_id),
'range': rangeset.Range('b', 'bb'),
'size': 0,
'is_del': 0, })
)
for b, expected in cases:
if b is None:
blk = BlockDesc()
else:
blk = BlockDesc(b)
self.assertEqual(expected, blk)
self.assertRaises(ValueError, BlockDesc, is_del='a')
self.assertRaises(ValueError, BlockDesc, size='a')
self.assertRaises(KeyError, BlockDesc, a=3)
def test_get_parities(self):
gid = 'g000640000000123'
g = BlockGroup(block_group_id=gid,
idcs=['a', 'b', 'c'],
config=_ec_config)
parities = g.get_parities(idc_index=0)
self.assertEqual([], parities)
base_parity = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
parity_idxes = ['0004', '0005']
for i, idx in enumerate(parity_idxes):
blkid = BlockID('dp', gid, idx, DriveID('idc000'
'c62d8736c7280002'), i)
parity = copy.deepcopy(base_parity)
parity['block_id'] = blkid
g.add_block(parity)
idxes = g.get_parity_indexes(idc_index=0)
self.assertEqual(parity_idxes, idxes)
parities = g.get_parities(idc_index=0)
idxes = []
for p in parities:
idx = BlockID(p['block_id']).block_index
idxes.append(idx)
self.assertEqual(parity_idxes, idxes)
def test_get_block_idc(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertEqual('a', g.get_block_idc('0000'))
self.assertEqual('b', g.get_block_idc('0100'))
self.assertEqual('c', g.get_block_idc('0200'))
d0 = BlockDesc({
'block_id':
BlockID('d0', 'g000640000000123', '0000',
DriveID('c62d8736c7280002'), 1),
'size':
1000,
'range': ['0a', '0b'],
'is_del':
0
})
g.add_block(d0)
self.assertEqual('a', g.get_block_idc('0000'))
def test_ref(self):
block_id = 'd1g0006300000001230101idc000c62d8736c72800020000000001'
blk = BlockDesc({
'block_id': block_id,
'size': 1000,
'range': ['0a', '0b'],
'ts_range': ["1235", "456"],
'ref_num': 1,
'is_del': 0,
'mtime': 1,
})
blk.add_ref()
blk.add_ref()
self.assertEqual(blk['ref_num'], 3)
blk.rm_ref()
self.assertEqual(blk['ref_num'], 2)
self.assertRaises(ValueError, blk.mark_del)
blk.rm_ref()
blk.rm_ref()
self.assertEqual(blk['ref_num'], 0)
self.assertTrue(blk.can_del())
self.assertRaises(ValueError, blk.rm_ref)
blk.mark_del()
def test_add_block(self):
region_cases = (
(
{},
(['a', 'c'], BlockDesc(), None),
{'idc': '', 'range': None, 'levels': [
[['a', 'c', BlockDesc()]],
]},
None,
),
(
{'idc': 'test', 'range': ['a', 'z'], 'levels': [
[['a', 'b', BlockDesc(size=1)], ['b', 'c', BlockDesc(size=2)]],
]},
(['c', 'd'], BlockDesc(), None),
{'idc': 'test', 'range': ['a', 'z'], 'levels': [
[['a', 'b', BlockDesc(size=1)], ['b', 'c', BlockDesc(size=2)]],
[['c', 'd', BlockDesc()]],
]},
None,
),
(
{'idc': 'test', 'range': ['a', 'z'], 'levels': [
[['a', 'b', BlockDesc(size=1)]],
[['b', 'c', BlockDesc(size=2)]],
]},
(['c', 'd'], BlockDesc(), 0),
{'idc': 'test', 'range': ['a', 'z'], 'levels': [
[['a', 'b', BlockDesc(size=1)], ['c', 'd', BlockDesc()]],
[['b', 'c', BlockDesc(size=2)]],
]},
None,
),
(
{'idc': 'test', 'range': ['a', 'z'], 'levels': [
[['a', 'b', BlockDesc(size=1)]],
[['b', 'c', BlockDesc(size=2)]],
]},
(['c', 'd'], BlockDesc(), 1),
{'idc': 'test', 'range': ['a', 'z'], 'levels': [
[['a', 'b', BlockDesc(size=1)]],
[['b', 'c', BlockDesc(size=2)], ['c', 'd', BlockDesc()]],
]},
None,
),
(
{'idc': 'test', 'range': ['a', 'z'], 'levels': [
[['a', 'b', BlockDesc(size=1)]],
[['b', 'c', BlockDesc(size=2)]],
]},
(['c', 'd'], BlockDesc(), 2),
{'idc': 'test', 'range': ['a', 'z'], 'levels': [
[['a', 'b', BlockDesc(size=1)]],
[['b', 'c', BlockDesc(size=2)]],
[['c', 'd', BlockDesc()]],
]},
None,
),
(
{'idc': 'test', 'range': ['a', 'z'], 'levels': [
[['a', 'b', BlockDesc(size=1)]],
[['b', 'c', BlockDesc(size=2)]],
]},
(['c', 'd'], BlockDesc(), 3),
{'idc': 'test', 'range': ['a', 'z'], 'levels': [
[['a', 'b', BlockDesc(size=1)]],
[['b', 'c', BlockDesc(size=2)]],
]},
LevelOutOfBound,
),
)
for case, args, excepted, err in region_cases:
region = Region(case)
if err is not None:
self.assertRaises(err, region.add_block, *args)
continue
region.add_block(*args)
self.assertEqual(excepted, region)
class TestBlockGroup(unittest.TestCase):
def setUp(self):
self.foo_block = BlockDesc({
'block_id':
BlockID('d0', 'g000640000000123', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1),
'size':
1000,
'range': ['0a', '0b'],
'ts_range':
None,
'ref_num':
1,
'is_del':
0
})
def test_new(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertEqual(_empty_group, g)
# test lacking of arg
self.assertRaises(TypeError,
BlockGroup,
block_group_id='g000640000000123',
idcs=[])
self.assertRaises(TypeError,
BlockGroup,
block_group_id='g000640000000123',
config=_ec_config)
self.assertRaises(TypeError, BlockGroup, idcs=[], config=_ec_config)
def test_json(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
rst = utfjson.dump(g)
expected = (
'{"config": {"in_idc": [4, 2], "ec_policy": "lrc", "cross_idc": [2, 1], '
'"data_replica": 3}, "blocks": {}, "idcs": ["a", "b", "c"], '
'"block_group_id": "g000640000000123"}')
self.assertEqual(expected, rst)
loaded = BlockGroup(utfjson.load(rst))
self.assertEqual(g, loaded)
def test_new_deref_config(self):
cnf = copy.deepcopy(_ec_config)
b = BlockGroup(block_group_id='g000640000000123',
config=cnf,
idcs=['a', 'b', 'c'])
a = copy.deepcopy(b['config'])
b['config']['in_idc'] = [10, 11]
self.assertNotEqual(a, b)
a = copy.deepcopy(b['config'])
b['config']['cross_idc'] = [10, 11]
self.assertNotEqual(a, b)
a = copy.deepcopy(b['config'])
b['config']['ec_policy'] = 'foo'
self.assertNotEqual(a, b)
a = copy.deepcopy(b['config'])
b['config']['data_replica'] = 100
self.assertNotEqual(a, b)
def test_get_block(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
block = g.get_block('0000', raise_error=False)
self.assertIsNone(block)
block = g.get_block('9999', raise_error=False)
self.assertIsNone(block)
with self.assertRaises(BlockNotFoundError):
g.get_block('9999', raise_error=True)
g.add_block(self.foo_block)
block = g.get_block(self.foo_block['block_id'].block_index)
self.assertDictEqual(self.foo_block, block)
with self.assertRaises(BlockNotFoundError):
g.get_block('0002', raise_error=True)
with self.assertRaises(ValueError):
g.get_block('d0g0006400000001230000c62d2')
def test_mark_delete_block(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
g.add_block(self.foo_block)
self.foo_block.add_ref()
self.assertEqual(2, self.foo_block['ref_num'])
self.assertEqual(0, self.foo_block['is_del'])
del_blk = g.mark_delete_block('0000')
self.assertIsNone(del_blk)
self.assertEqual(1, self.foo_block['ref_num'])
self.assertEqual(0, self.foo_block['is_del'])
del_blk = g.mark_delete_block('0000')
self.assertEqual(del_blk['ref_num'], 0)
self.assertEqual(1, del_blk['is_del'])
self.assertDictEqual(del_blk, g.get_block('0000'))
self.assertTrue(int(time.time()) - del_blk["mtime"] < 3)
self.assertRaises(BlockNotFoundError, g.mark_delete_block, '9999')
def test_mark_delete_block_byid(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
g.add_block(self.foo_block)
self.foo_block.add_ref()
self.assertEqual(2, self.foo_block['ref_num'])
self.assertEqual(0, self.foo_block['is_del'])
del_blk = g.mark_delete_block_byid(self.foo_block['block_id'])
self.assertIsNone(del_blk)
self.assertEqual(1, self.foo_block['ref_num'])
self.assertEqual(0, self.foo_block['is_del'])
self.assertDictEqual(self.foo_block,
g.get_block_byid(self.foo_block['block_id']))
del_blk = g.mark_delete_block_byid(self.foo_block['block_id'])
self.assertEqual(del_blk['ref_num'], 0)
self.assertEqual(1, del_blk['is_del'])
self.assertDictEqual(del_blk,
g.get_block_byid(self.foo_block['block_id']))
self.assertTrue(int(time.time()) - del_blk["mtime"] < 3)
fake_bid = BlockID('d0', 'g000640000000125', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1)
self.assertRaises(BlockNotFoundError, g.mark_delete_block_byid,
fake_bid)
def test_delete_block(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertIsNone(g.get_block('0000', raise_error=False))
g.add_block(self.foo_block)
self.assertIsNotNone(g.get_block('0000'))
self.foo_block.add_ref()
del_blk = g.delete_block('0000')
self.assertIsNotNone(g.get_block('0000', raise_error=False))
del_blk = g.delete_block('0000')
self.assertIsNone(g.get_block('0000', raise_error=False))
self.assertDictEqual(self.foo_block, del_blk)
self.assertRaises(BlockNotFoundError, g.delete_block, '0000')
def test_replace_block(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
prev = g.add_block(self.foo_block)
self.assertIsNone(prev)
block = g.get_block('0000')
self.assertEqual(0, block['is_del'])
prev = g.add_block(self.foo_block, replace=True)
self.assertEqual(self.foo_block, prev)
self.assertRaises(BlockExists, g.add_block, self.foo_block)
self.assertRaises(BlockExists,
g.add_block,
self.foo_block,
replace=False)
def test_get_free_block_index(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
g.add_block(self.foo_block)
self.assertDictEqual(
{
'a': ['0001', '0002', '0003'],
'b': ['0100', '0101', '0102', '0103']
}, g.get_free_block_indexes('d0'))
self.assertDictEqual({
'a': ['0004', '0005'],
'b': ['0104', '0105']
}, g.get_free_block_indexes('dp'))
self.assertDictEqual({
'c': ['0200', '0201', '0202', '0203'],
}, g.get_free_block_indexes('x0'))
self.assertDictEqual({
'c': ['0204', '0205'],
}, g.get_free_block_indexes('xp'))
self.assertDictEqual(
{
'a': ['0001', '0002', '0003'],
'b': ['0100', '0101', '0102', '0103'],
'c': [],
}, g.get_free_block_indexes('d0', get_all=True))
self.assertDictEqual(
{
'a': ['0004', '0005'],
'b': ['0104', '0105'],
'c': [],
}, g.get_free_block_indexes('dp', get_all=True))
self.assertDictEqual(
{
'a': [],
'b': [],
'c': ['0200', '0201', '0202', '0203'],
}, g.get_free_block_indexes('x0', get_all=True))
self.assertDictEqual({
'a': [],
'b': [],
'c': ['0204', '0205'],
}, g.get_free_block_indexes('xp', get_all=True))
def test_get_block_type(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertEqual('d0', g.get_block_type('0000'))
self.assertEqual('dp', g.get_block_type('0004'))
self.assertEqual('d1', g.get_block_type('0006'))
self.assertEqual('d0', g.get_block_type('0100'))
self.assertEqual('dp', g.get_block_type('0104'))
self.assertEqual('d1', g.get_block_type('0106'))
self.assertEqual('x0', g.get_block_type('0200'))
self.assertEqual('xp', g.get_block_type('0204'))
self.assertRaises(BlockTypeNotSupported, g.get_block_type, '0299')
self.assertRaises(BlockTypeNotSupported, g.get_block_type, '0900')
def test_get_block_idc(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertEqual('a', g.get_block_idc('0000'))
self.assertEqual('b', g.get_block_idc('0100'))
self.assertEqual('c', g.get_block_idc('0200'))
d0 = BlockDesc({
'block_id':
BlockID('d0', 'g000640000000123', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1),
'size':
1000,
'range': ['0a', '0b'],
'is_del':
0
})
g.add_block(d0)
self.assertEqual('a', g.get_block_idc('0000'))
def test_get_replica_index_not_include_me(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertEqual(['0006', '0010'],
g.get_replica_indexes('0000', include_me=False))
self.assertEqual(['0000', '0010'],
g.get_replica_indexes('0006', include_me=False))
self.assertEqual(['0000', '0006'],
g.get_replica_indexes('0010', include_me=False))
with self.assertRaises(BlockTypeNotSupportReplica):
g.get_replica_indexes('0004', include_me=False)
with self.assertRaises(BlockTypeNotSupportReplica):
g.get_replica_indexes('0204', include_me=False)
def test_classify_blocks(self):
gid = 'g000640000000123'
g = BlockGroup(block_group_id=gid,
idcs=['a', 'b', 'c'],
config=_ec_config)
blks = g.classify_blocks(0, only_primary=True)
self.assertEqual([], blks['ec'] + blks['replica'] + blks['mark_del'])
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
ec_blk_idxes = ['0000', '0001']
replica_blk_idxes = ['0002', '0008', '0012']
mark_del_idxes = ['0003', '0004']
for i, idx in enumerate(ec_blk_idxes + replica_blk_idxes +
mark_del_idxes):
typ = g.get_block_type(idx)
blkid = BlockID(typ, gid, idx, DriveID('idc000'
'c62d8736c7280002'), i)
blk = copy.deepcopy(base_blk)
blk['block_id'] = blkid
if idx in mark_del_idxes:
blk['is_del'] = 1
g.add_block(blk)
for only_primary in (True, False):
blks = g.classify_blocks(0, only_primary)
blk_idxes = []
for blk in blks['ec'] + blks['replica'] + blks['mark_del']:
idx = BlockID(blk['block_id']).block_index
blk_idxes.append(idx)
expect_ids = copy.deepcopy(ec_blk_idxes)
#'0004' in ec_blk_idxes is parity, so should not in mark_del
if only_primary is True:
expect_ids += replica_blk_idxes[:1] + mark_del_idxes[:1]
else:
expect_ids += replica_blk_idxes + mark_del_idxes[:1]
self.assertEqual(expect_ids, blk_idxes)
def test_get_parities(self):
gid = 'g000640000000123'
g = BlockGroup(block_group_id=gid,
idcs=['a', 'b', 'c'],
config=_ec_config)
parities = g.get_parities(idc_index=0)
self.assertEqual([], parities)
base_parity = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
parity_idxes = ['0004', '0005']
for i, idx in enumerate(parity_idxes):
blkid = BlockID('dp', gid, idx, DriveID('idc000'
'c62d8736c7280002'), i)
parity = copy.deepcopy(base_parity)
parity['block_id'] = blkid
g.add_block(parity)
idxes = g.get_parity_indexes(idc_index=0)
self.assertEqual(parity_idxes, idxes)
parities = g.get_parities(idc_index=0)
idxes = []
for p in parities:
idx = BlockID(p['block_id']).block_index
idxes.append(idx)
self.assertEqual(parity_idxes, idxes)
def make_test_block_group(self, blk_idxes, config=None):
gid = 'g000640000000123'
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'ts_range': ["123", "456"],
'ref_num': 1,
'is_del': 0
})
if config is None:
config = _ec_config
num_idcs = sum(config['cross_idc'])
idcs = ['idc' + (str(i).rjust(3, '0')) for i in range(num_idcs)]
bg = BlockGroup(block_group_id=gid, idcs=idcs, config=config)
for i, bi in enumerate(blk_idxes):
bi = BlockIndex(bi)
typ = bg.get_block_type(bi)
drive_id = DriveID(idcs[int(bi[0])] + 'c62d8736c7280002')
blkid = BlockID(typ, gid, bi, drive_id, i)
blk = copy.deepcopy(base_blk)
blk['block_id'] = blkid
bg.add_block(blk)
return bg
def test_is_ec_block(self):
idc_idx = 0
ec_blk_idxes = ['0000', '0001', '0005']
replica_blk_idxes = ['0002', '0008', '0012']
bg = self.make_test_block_group(ec_blk_idxes + replica_blk_idxes)
with self.assertRaises(BlockNotFoundError):
gid = 'g000640000000123'
bid = BlockID('dp', gid, '0001',
DriveID('idc000'
'ab2d8736c7280002'), 0)
bg.is_ec_block(bid)
act_ec_blk_idxes = []
nr_data, nr_parity = bg['config']['in_idc']
for i in range(0, nr_data + nr_parity):
bi = BlockIndex(idc_idx, i)
blk = bg.get_block(bi, raise_error=False)
if blk is None:
continue
if bg.is_ec_block(blk['block_id']):
act_ec_blk_idxes.append(bi)
self.assertListEqual(ec_blk_idxes, act_ec_blk_idxes)
def test_get_ec_blocks(self):
idc_idx = 0
ec_blk_idxes = ['0000', '0001']
replica_blk_idxes = ['0002', '0008', '0012']
bg = self.make_test_block_group(ec_blk_idxes + replica_blk_idxes)
ec_blks = bg.indexes_to_blocks(ec_blk_idxes)
act_ec_blks = bg.get_ec_blocks(idc_idx)
self.assertListEqual(ec_blks, act_ec_blks)
def test_get_ec_broken_blocks(self):
idc_idx = 0
ec_blk_idxes = ['0000', '0001', '0003']
replica_blk_idxes = ['0002', '0008', '0012']
bg = self.make_test_block_group(ec_blk_idxes + replica_blk_idxes)
broken_blk_idxes = ec_blk_idxes[1:]
broken_blks = bg.indexes_to_blocks(broken_blk_idxes)
broken_ec_bids = [blk['block_id'] for blk in broken_blks]
act_broken_blks = bg.get_ec_broken_blocks(idc_idx, broken_ec_bids)
self.assertListEqual(broken_blks, act_broken_blks)
def test_get_ec_block_ids(self):
idc_idx = 0
ec_blk_idxes = ['0000', '0001', '0003']
replica_blk_idxes = ['0002', '0008', '0012']
bg = self.make_test_block_group(ec_blk_idxes + replica_blk_idxes)
ec_blks = bg.indexes_to_blocks(ec_blk_idxes)
ec_bids = [blk['block_id'] for blk in ec_blks]
act_ec_bids = bg.get_ec_block_ids(idc_idx)
self.assertListEqual(ec_bids, act_ec_bids)
def test_get_replica_blocks(self):
ec_blk_idxes = ['0000', '0001', '0003']
replica_blk_idxes = ['0002', '0008', '0012']
bg = self.make_test_block_group(ec_blk_idxes + replica_blk_idxes)
replica_blks = bg.indexes_to_blocks(replica_blk_idxes)
for blk in replica_blks:
bid = blk['block_id']
act_replica_blks = bg.get_replica_blocks(bid)
self.assertListEqual(replica_blks, act_replica_blks)
_replica_blks = copy.deepcopy(replica_blks)
_replica_blks.remove(blk)
act_replica_blks = bg.get_replica_blocks(bid, include_me=False)
self.assertListEqual(_replica_blks, act_replica_blks)
fake_bid = BlockID('d0', 'g000640000000125', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1)
self.assertIsNone(bg.get_replica_blocks(fake_bid, raise_error=False))
self.assertRaises(BlockNotFoundError,
bg.get_replica_blocks,
fake_bid,
raise_error=True)
self.assertRaises(BlockNotFoundError, bg.get_replica_blocks, fake_bid)
def test_get_block_byid(self):
blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']
bg = self.make_test_block_group(blk_idxes)
blks = bg.indexes_to_blocks(blk_idxes)
bids = [blk['block_id'] for blk in blks]
act_blks = []
for bid in bids:
act_blks.append(bg.get_block_byid(bid))
self.assertListEqual(blks, act_blks)
fake_bid = BlockID('d0', 'g000640000000125', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1)
self.assertIsNone(bg.get_block_byid(fake_bid, raise_error=False))
self.assertRaises(BlockNotFoundError, bg.get_block_byid, fake_bid,
True)
self.assertRaises(BlockNotFoundError, bg.get_block_byid, fake_bid)
def test_unlink_block_byid(self):
blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']
bg = self.make_test_block_group(blk_idxes)
blks = bg.indexes_to_blocks(blk_idxes)
bids = [blk['block_id'] for blk in blks]
bg.link_block_byid(bids[1])
self.assertEqual(blks[1]['ref_num'], 2)
del_blk = bg.delete_block_byid(bids[1])
self.assertIsNone(del_blk)
del_blk = bg.mark_delete_block_byid(bids[1])
self.assertEqual(del_blk["is_del"], 1)
del_blk = bg.delete_block_byid(bids[1])
self.assertDictEqual(del_blk, blks[1])
self.assertRaises(BlockNotFoundError, bg.delete_block_byid, bids[1])
blks.pop(1)
act_blks = bg.indexes_to_blocks(blk_idxes)
act_blks = [blk for blk in act_blks if blk is not None]
self.assertListEqual(blks, act_blks)
def test_link_block(self):
blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']
bg = self.make_test_block_group(blk_idxes)
blks = bg.indexes_to_blocks(blk_idxes)
args = [blks[1], False, False]
self.assertRaises(BlockExists, bg.add_block, *args)
act_blks = bg.indexes_to_blocks(blk_idxes)
self.assertListEqual(blks, act_blks)
bg.link_block(blk_idxes[1])
self.assertEqual(blks[1]['ref_num'], 2)
bg.unlink_block(blk_idxes[1])
self.assertDictEqual(blks[1], bg.add_block(blks[1], allow_exist=True))
act_blks = bg.indexes_to_blocks(blk_idxes)
self.assertListEqual(blks, act_blks)
def test_add_block(self):
blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']
bg = self.make_test_block_group(blk_idxes)
blks = bg.indexes_to_blocks(blk_idxes)
args = [blks[1], False, False]
self.assertRaises(BlockExists, bg.add_block, *args)
act_blks = bg.indexes_to_blocks(blk_idxes)
self.assertListEqual(blks, act_blks)
self.assertDictEqual(blks[1], bg.add_block(blks[1], allow_exist=True))
act_blks = bg.indexes_to_blocks(blk_idxes)
self.assertListEqual(blks, act_blks)
def test_get_idc_blocks(self):
idc0 = ['0000', '0001', '0002']
idc1 = ['0103', '0108', '0112']
blk_idxes = idc0 + idc1
bg = self.make_test_block_group(blk_idxes)
idc0_blks = bg.indexes_to_blocks(idc0)
act_idc0_blks = bg.get_idc_blocks(0)
self.assertListEqual(idc0_blks, act_idc0_blks)
idc1_blks = bg.indexes_to_blocks(idc1)
act_idc1_blks = bg.get_idc_blocks(1)
self.assertListEqual(idc1_blks, act_idc1_blks)
for blk in idc1:
bg.mark_delete_block(blk)
act_idc1_blks = bg.get_idc_blocks(1, is_del=False)
self.assertListEqual([], act_idc1_blks)
act_idc1_blks = bg.get_idc_blocks(1, is_del=True)
self.assertListEqual(idc1_blks, act_idc1_blks)
act_idc2_blks = bg.get_idc_blocks(2)
self.assertListEqual([], act_idc2_blks)
def test_get_get_blocks(self):
idc0 = ['0000', '0001', '0002']
idc1 = ['0103', '0108', '0112']
blk_idxes = idc0 + idc1
bg = self.make_test_block_group(blk_idxes)
blks = bg.indexes_to_blocks(blk_idxes)
act_blks = bg.get_blocks()
self.assertListEqual(blks, act_blks)
def test_block_type(self):
ec_idxes = ['0000', '0001', '0004', '0005']
replica_idxes = ['0002', '0008', '0012']
idxes = ec_idxes + replica_idxes
bg = self.make_test_block_group(idxes)
blks = bg.indexes_to_blocks(idxes)
self.assertTrue(BlockGroup.is_data(blks[0]['block_id']))
self.assertTrue(BlockGroup.is_data(blks[1]['block_id']))
self.assertTrue(BlockGroup.is_parity(blks[2]['block_id']))
self.assertTrue(BlockGroup.is_parity(blks[3]['block_id']))
self.assertTrue(BlockGroup.is_data(blks[4]['block_id']))
self.assertTrue(BlockGroup.is_replica(blks[5]['block_id']))
self.assertTrue(BlockGroup.is_replica(blks[6]['block_id']))
def test_get_d0_idcs(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertEqual(["a", "b"], g.get_d0_idcs())
def test_get_dtype_by_idc(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertEqual("d0", g.get_dtype_by_idc("a"))
self.assertEqual("d0", g.get_dtype_by_idc("b"))
self.assertEqual("x0", g.get_dtype_by_idc("c"))
self.assertRaises(AssertionError, g.get_dtype_by_idc, "d")
def test_move_down(self):
region_levels = [
[
['aa', 'ee', BlockDesc(size=1)],
['hh', 'hz', BlockDesc(size=2)],
['pp', 'zz', BlockDesc(size=3)],
['zz', None, BlockDesc(size=4)]
],
[
['cf', 'cz', BlockDesc(size=5)],
['mm', 'oo', BlockDesc(size=6)],
['oo', 'qq', BlockDesc(size=7)]
],
[
['aa', 'bb', BlockDesc(size=8)],
['cc', 'cd', BlockDesc(size=9)],
['ee', 'ff', BlockDesc(size=10)]
],
[
['aa', 'ab', BlockDesc(size=11)],
['az', 'bb', BlockDesc(size=12)],
['za', None, BlockDesc(size=13)]
],
[
['d', 'fz', BlockDesc(size=14)]
],
]
excepted_region_levels = [
[
['aa', 'ee', BlockDesc(size=1)],
['ee', 'ff', BlockDesc(size=10)],
['hh', 'hz', BlockDesc(size=2)],
['mm', 'oo', BlockDesc(size=6)],
['pp', 'zz', BlockDesc(size=3)],
['zz', None, BlockDesc(size=4)]
],
[
['aa', 'bb', BlockDesc(size=8)],
['cc', 'cd', BlockDesc(size=9)],
['cf', 'cz', BlockDesc(size=5)],
['d', 'fz', BlockDesc(size=14)],
['oo', 'qq', BlockDesc(size=7)],
['za', None, BlockDesc(size=13)]
],
[
['aa', 'ab', BlockDesc(size=11)],
['az', 'bb', BlockDesc(size=12)]
],
]
excepted_moved_blocks = [
(1, 0, ['mm', 'oo', BlockDesc(size=6)]),
(2, 1, ['aa', 'bb', BlockDesc(size=8)]),
(2, 1, ['cc', 'cd', BlockDesc(size=9)]),
(2, 0, ['ee', 'ff', BlockDesc(size=10)]),
(3, 2, ['aa', 'ab', BlockDesc(size=11)]),
(3, 2, ['az', 'bb', BlockDesc(size=12)]),
(3, 1, ['za', None, BlockDesc(size=13)]),
(4, 1, ['d', 'fz', BlockDesc(size=14)]),
]
region = Region(levels=region_levels)
moved_blocks = region.move_down()
self.assertEqual(excepted_moved_blocks, moved_blocks)
self.assertEqual(excepted_region_levels, region['levels'])
region_levels = [
[['aa', 'ee', BlockDesc(size=1)], ['ee', 'ff', BlockDesc(size=10)], ['hh', 'hz', BlockDesc(size=2)]],
[['aa', 'yy', BlockDesc(size=8)]]
]
region = Region(levels=region_levels)
moved_blocks = region.move_down()
self.assertEqual([], moved_blocks)
self.assertEqual(region_levels, region['levels'])
