def test_mark_delete_block_byid(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
g.add_block(self.foo_block)
self.foo_block.add_ref()
self.assertEqual(2, self.foo_block['ref_num'])
self.assertEqual(0, self.foo_block['is_del'])
del_blk = g.mark_delete_block_byid(self.foo_block['block_id'])
self.assertIsNone(del_blk)
self.assertEqual(1, self.foo_block['ref_num'])
self.assertEqual(0, self.foo_block['is_del'])
self.assertDictEqual(self.foo_block,
g.get_block_byid(self.foo_block['block_id']))
del_blk = g.mark_delete_block_byid(self.foo_block['block_id'])
self.assertEqual(del_blk['ref_num'], 0)
self.assertEqual(1, del_blk['is_del'])
self.assertDictEqual(del_blk,
g.get_block_byid(self.foo_block['block_id']))
self.assertTrue(int(time.time()) - del_blk["mtime"] < 3)
fake_bid = BlockID('d0', 'g000640000000125', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1)
self.assertRaises(BlockNotFoundError, g.mark_delete_block_byid,
fake_bid)
def make_test_block_group(self, blk_idxes, config=None):
gid = 'g000640000000123'
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
if config is None:
config = _ec_config
num_idcs = sum(config['cross_idc'])
idcs = ['idc' + (str(i).rjust(3, '0')) for i in range(num_idcs)]
bg = BlockGroup(block_group_id=gid, idcs=idcs, config=config)
for i, bi in enumerate(blk_idxes):
bi = BlockIndex(bi)
typ = bg.get_block_type(bi)
drive_id = DriveID(idcs[int(bi[0])] + 'c62d8736c7280002')
blkid = BlockID(typ, gid, bi, drive_id, i)
blk = copy.deepcopy(base_blk)
blk['block_id'] = blkid
bg.add_block(blk)
return bg
def make_test_block_group(self, blk_idxes, config=None):
gid = 'g000640000000123'
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
if config is None:
config = _ec_config
num_idcs = sum(config['cross_idc'])
idcs = ['idc' + (str(i).rjust(3, '0')) for i in range(num_idcs)]
bg = BlockGroup(block_group_id=gid, idcs=idcs, config=config)
for i, bi in enumerate(blk_idxes):
bi = BlockIndex(bi)
typ = bg.get_block_type(bi)
drive_id = DriveID(idcs[int(bi[0])] + 'c62d8736c7280002')
blkid = BlockID(typ, gid, bi, drive_id, i)
blk = copy.deepcopy(base_blk)
blk['block_id'] = blkid
bg.add_block(blk)
return bg
def test_mark_delete_block(self):
g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config)
g.add_block(self.foo_block)
g.mark_delete_block('0000')
block = g.get_block('0000')
self.assertEqual(1, block['is_del'])
self.assertRaises(BlockNotFoundError, g.mark_delete_block, '9999')
def test_classify_blocks(self):
gid = 'g000640000000123'
g = BlockGroup(block_group_id=gid,
idcs=['a', 'b', 'c'],
config=_ec_config)
blks = g.classify_blocks(0, only_primary=True)
self.assertEqual([], blks['ec'] + blks['replica'] + blks['mark_del'])
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
ec_blk_idxes = ['0000', '0001']
replica_blk_idxes = ['0002', '0008', '0012']
mark_del_idxes = ['0003', '0004']
for i, idx in enumerate(ec_blk_idxes + replica_blk_idxes +
mark_del_idxes):
typ = g.get_block_type(idx)
blkid = BlockID(typ, gid, idx, DriveID('idc000'
'c62d8736c7280002'), i)
blk = copy.deepcopy(base_blk)
blk['block_id'] = blkid
if idx in mark_del_idxes:
blk['is_del'] = 1
g.add_block(blk)
for only_primary in (True, False):
blks = g.classify_blocks(0, only_primary)
blk_idxes = []
for blk in blks['ec'] + blks['replica'] + blks['mark_del']:
idx = BlockID(blk['block_id']).block_index
blk_idxes.append(idx)
expect_ids = copy.deepcopy(ec_blk_idxes)
#'0004' in ec_blk_idxes is parity, so should not in mark_del
if only_primary is True:
expect_ids += replica_blk_idxes[:1] + mark_del_idxes[:1]
else:
expect_ids += replica_blk_idxes + mark_del_idxes[:1]
self.assertEqual(expect_ids, blk_idxes)
def test_mark_delete_block(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
g.add_block(self.foo_block)
g.mark_delete_block('0000')
block = g.get_block('0000')
self.assertEqual(1, block['is_del'])
self.assertRaises(BlockNotFoundError, g.mark_delete_block, '9999')
def test_classify_blocks(self):
gid = 'g000640000000123'
g = BlockGroup(block_group_id=gid, idcs=['a', 'b', 'c'], config=_ec_config)
blks = g.classify_blocks(0, only_primary=True)
self.assertEqual([], blks['ec'] + blks['replica'] + blks['mark_del'])
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
ec_blk_idxes = ['0000', '0001']
replica_blk_idxes = ['0002', '0008', '0012']
mark_del_idxes = ['0003', '0004']
for i, idx in enumerate(ec_blk_idxes + replica_blk_idxes + mark_del_idxes):
typ = g.get_block_type(idx)
blkid = BlockID(typ, gid, idx, DriveID('idc000' 'c62d8736c7280002'), i)
blk = copy.deepcopy(base_blk)
blk['block_id'] = blkid
if idx in mark_del_idxes:
blk['is_del'] = 1
g.add_block(blk)
for only_primary in (True, False):
blks = g.classify_blocks(0, only_primary)
blk_idxes = []
for blk in blks['ec'] + blks['replica'] + blks['mark_del']:
idx = BlockID(blk['block_id']).block_index
blk_idxes.append(idx)
expect_ids = copy.deepcopy(ec_blk_idxes)
#'0004' in ec_blk_idxes is parity, so should not in mark_del
if only_primary is True:
expect_ids += replica_blk_idxes[:1] + mark_del_idxes[:1]
else:
expect_ids += replica_blk_idxes + mark_del_idxes[:1]
self.assertEqual(expect_ids, blk_idxes)
def test_get_free_block_index(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
g.add_block(self.foo_block)
self.assertDictEqual(
{
'a': ['0001', '0002', '0003'],
'b': ['0100', '0101', '0102', '0103']
}, g.get_free_block_indexes('d0'))
self.assertDictEqual({
'a': ['0004', '0005'],
'b': ['0104', '0105']
}, g.get_free_block_indexes('dp'))
self.assertDictEqual({
'c': ['0200', '0201', '0202', '0203'],
}, g.get_free_block_indexes('x0'))
self.assertDictEqual({
'c': ['0204', '0205'],
}, g.get_free_block_indexes('xp'))
self.assertDictEqual(
{
'a': ['0001', '0002', '0003'],
'b': ['0100', '0101', '0102', '0103'],
'c': [],
}, g.get_free_block_indexes('d0', get_all=True))
self.assertDictEqual(
{
'a': ['0004', '0005'],
'b': ['0104', '0105'],
'c': [],
}, g.get_free_block_indexes('dp', get_all=True))
self.assertDictEqual(
{
'a': [],
'b': [],
'c': ['0200', '0201', '0202', '0203'],
}, g.get_free_block_indexes('x0', get_all=True))
self.assertDictEqual({
'a': [],
'b': [],
'c': ['0204', '0205'],
}, g.get_free_block_indexes('xp', get_all=True))
def test_delete_block(self):
g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config)
self.assertIsNone(g.get_block('0000'))
g.add_block(self.foo_block)
self.assertIsNotNone(g.get_block('0000'))
g.delete_block('0000')
self.assertIsNone(g.get_block('0000'))
g.delete_block('0000')
self.assertIsNone(g.get_block('0000'))
def test_get_free_block_index(self):
g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config)
g.add_block(self.foo_block)
self.assertDictEqual({'a': ['0001', '0002', '0003'],
'b': ['0100', '0101', '0102', '0103']},
g.get_free_block_indexes('d0'))
self.assertDictEqual({'a': ['0004', '0005'],
'b': ['0104', '0105']},
g.get_free_block_indexes('dp'))
self.assertDictEqual({'c': ['0200', '0201', '0202', '0203'], },
g.get_free_block_indexes('x0'))
self.assertDictEqual({'c': ['0204', '0205'], },
g.get_free_block_indexes('xp'))
self.assertDictEqual(
{
'a': ['0001', '0002', '0003'],
'b': ['0100', '0101', '0102', '0103'],
'c': [],
},
g.get_free_block_indexes('d0', get_all=True))
self.assertDictEqual(
{
'a': ['0004', '0005'],
'b': ['0104', '0105'],
'c': [],
},
g.get_free_block_indexes('dp', get_all=True))
self.assertDictEqual(
{
'a': [],
'b': [],
'c': ['0200', '0201', '0202', '0203'],
},
g.get_free_block_indexes('x0', get_all=True))
self.assertDictEqual(
{
'a': [],
'b': [],
'c': ['0204', '0205'],
},
g.get_free_block_indexes('xp', get_all=True))
def test_delete_block(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertIsNone(g.get_block('0000'))
g.add_block(self.foo_block)
self.assertIsNotNone(g.get_block('0000'))
g.delete_block('0000')
self.assertIsNone(g.get_block('0000'))
g.delete_block('0000')
self.assertIsNone(g.get_block('0000'))
def test_replace_block(self):
g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config)
prev = g.add_block(self.foo_block)
self.assertIsNone(prev)
block = g.get_block('0000')
self.assertEqual(0, block['is_del'])
prev = g.add_block(self.foo_block, replace=True)
self.assertEqual(self.foo_block, prev)
self.assertRaises(BlockExists, g.add_block, self.foo_block)
self.assertRaises(BlockExists, g.add_block, self.foo_block, replace=False)
def test_get_block_idc(self):
g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config)
self.assertEqual('a', g.get_block_idc('0000'))
self.assertEqual('b', g.get_block_idc('0100'))
self.assertEqual('c', g.get_block_idc('0200'))
d0 = BlockDesc({
'block_id': BlockID('d0', 'g000640000000123', '0000',
DriveID('idc000' 'c62d8736c7280002'), 1),
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
g.add_block(d0)
self.assertEqual('a', g.get_block_idc('0000'))
def test_delete_block(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertIsNone(g.get_block('0000', raise_error=False))
g.add_block(self.foo_block)
self.assertIsNotNone(g.get_block('0000'))
self.foo_block.add_ref()
del_blk = g.delete_block('0000')
self.assertIsNotNone(g.get_block('0000', raise_error=False))
del_blk = g.delete_block('0000')
self.assertIsNone(g.get_block('0000', raise_error=False))
self.assertDictEqual(self.foo_block, del_blk)
self.assertRaises(BlockNotFoundError, g.delete_block, '0000')
def test_replace_block(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
prev = g.add_block(self.foo_block)
self.assertIsNone(prev)
block = g.get_block('0000')
self.assertEqual(0, block['is_del'])
prev = g.add_block(self.foo_block, replace=True)
self.assertEqual(self.foo_block, prev)
self.assertRaises(BlockExists, g.add_block, self.foo_block)
self.assertRaises(BlockExists,
g.add_block,
self.foo_block,
replace=False)
def test_get_parities(self):
gid = 'g000640000000123'
g = BlockGroup(block_group_id=gid,
idcs=['a', 'b', 'c'],
config=_ec_config)
parities = g.get_parities(idc_index=0)
self.assertEqual([], parities)
base_parity = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
parity_idxes = ['0004', '0005']
for i, idx in enumerate(parity_idxes):
blkid = BlockID('dp', gid, idx, DriveID('idc000'
'c62d8736c7280002'), i)
parity = copy.deepcopy(base_parity)
parity['block_id'] = blkid
g.add_block(parity)
idxes = g.get_parity_indexes(idc_index=0)
self.assertEqual(parity_idxes, idxes)
parities = g.get_parities(idc_index=0)
idxes = []
for p in parities:
idx = BlockID(p['block_id']).block_index
idxes.append(idx)
self.assertEqual(parity_idxes, idxes)
def test_get_block_idc(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertEqual('a', g.get_block_idc('0000'))
self.assertEqual('b', g.get_block_idc('0100'))
self.assertEqual('c', g.get_block_idc('0200'))
d0 = BlockDesc({
'block_id':
BlockID('d0', 'g000640000000123', '0000',
DriveID('c62d8736c7280002'), 1),
'size':
1000,
'range': ['0a', '0b'],
'is_del':
0
})
g.add_block(d0)
self.assertEqual('a', g.get_block_idc('0000'))
def test_get_block(self):
g = BlockGroup(block_group_id='g000640000000123', idcs=['a', 'b', 'c'], config=_ec_config)
block = g.get_block('0000')
self.assertIsNone(block)
block = g.get_block('9999')
self.assertIsNone(block)
with self.assertRaises(BlockNotFoundError):
g.get_block('9999', raise_error=True)
g.add_block(self.foo_block)
block = g.get_block(self.foo_block['block_id'].block_index)
self.assertDictEqual(self.foo_block, block)
with self.assertRaises(BlockNotFoundError):
g.get_block('0002', raise_error=True)
with self.assertRaises(ValueError):
g.get_block('d0g0006400000001230000c62d2')
def test_get_parities(self):
gid = 'g000640000000123'
g = BlockGroup(block_group_id=gid, idcs=['a', 'b', 'c'], config=_ec_config)
parities = g.get_parities(idc_index=0)
self.assertEqual([], parities)
base_parity = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
parity_idxes = ['0004', '0005']
for i, idx in enumerate(parity_idxes):
blkid = BlockID('dp', gid, idx, DriveID('idc000' 'c62d8736c7280002'), i)
parity = copy.deepcopy(base_parity)
parity['block_id'] = blkid
g.add_block(parity)
idxes = g.get_parity_indexes(idc_index=0)
self.assertEqual(parity_idxes, idxes)
parities = g.get_parities(idc_index=0)
idxes = []
for p in parities:
idx = BlockID(p['block_id']).block_index
idxes.append(idx)
self.assertEqual(parity_idxes, idxes)
def test_get_block(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
block = g.get_block('0000')
self.assertIsNone(block)
block = g.get_block('9999')
self.assertIsNone(block)
with self.assertRaises(BlockNotFoundError):
g.get_block('9999', raise_error=True)
g.add_block(self.foo_block)
block = g.get_block(self.foo_block['block_id'].block_index)
self.assertDictEqual(self.foo_block, block)
with self.assertRaises(BlockNotFoundError):
g.get_block('0002', raise_error=True)
with self.assertRaises(ValueError):
g.get_block('d0g0006400000001230000c62d2')
def test_mark_delete_block(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
g.add_block(self.foo_block)
self.foo_block.add_ref()
self.assertEqual(2, self.foo_block['ref_num'])
self.assertEqual(0, self.foo_block['is_del'])
del_blk = g.mark_delete_block('0000')
self.assertIsNone(del_blk)
self.assertEqual(1, self.foo_block['ref_num'])
self.assertEqual(0, self.foo_block['is_del'])
del_blk = g.mark_delete_block('0000')
self.assertEqual(del_blk['ref_num'], 0)
self.assertEqual(1, del_blk['is_del'])
self.assertDictEqual(del_blk, g.get_block('0000'))
self.assertTrue(int(time.time()) - del_blk["mtime"] < 3)
self.assertRaises(BlockNotFoundError, g.mark_delete_block, '9999')