def test_drive_id(self):
cases = (
('112233445566', 1),
('112233445566', 10),
('112233445566', 100),
('112233445566', 999),
('aabbccddeeff', 1),
('aabbccddeeff', 10),
('aabbccddeeff', 100),
('aabbccddeeff', 999),
('1122ccddeeff', 1),
('1122ccddeeff', 10),
('1122ccddeeff', 100),
('1122ccddeeff', 999),
)
for sid, mp_idx in cases:
drive_id = DriveID(sid, mp_idx)
self.assertEqual(sid, drive_id.server_id)
self.assertEqual('%03d' % mp_idx, drive_id.mountpoint_index)
self.assertEqual(mp_idx, int(drive_id.mountpoint_index))
self.assertEqual('%s0%03d' % (sid[:12], mp_idx % 1000), drive_id)
drvid = DriveID(drive_id)
self.assertEqual(sid, drvid.server_id)
self.assertEqual(drvid, DriveID(drvid))
def test_drive_id_server_id(self):
for drive_id in (DriveID('112233445566',
1), DriveID('1122334455660001')):
dd(drive_id)
self.assertIsInstance(drive_id.server_id, str)
self.assertEqual('112233445566', drive_id.server_id)
self.assertEqual('1122334455660001', str(drive_id))
def test_drive_id_port(self):
for drive_id in (DriveID('idc000112233445566',
1), DriveID('idc0001122334455660001')):
dd(drive_id)
self.assertIsInstance(drive_id.port, int)
self.assertEqual(6001, drive_id.port)
self.assertEqual('idc0001122334455660001', str(drive_id))
def test_mark_delete_block_byid(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
g.add_block(self.foo_block)
self.foo_block.add_ref()
self.assertEqual(2, self.foo_block['ref_num'])
self.assertEqual(0, self.foo_block['is_del'])
del_blk = g.mark_delete_block_byid(self.foo_block['block_id'])
self.assertIsNone(del_blk)
self.assertEqual(1, self.foo_block['ref_num'])
self.assertEqual(0, self.foo_block['is_del'])
self.assertDictEqual(self.foo_block,
g.get_block_byid(self.foo_block['block_id']))
del_blk = g.mark_delete_block_byid(self.foo_block['block_id'])
self.assertEqual(del_blk['ref_num'], 0)
self.assertEqual(1, del_blk['is_del'])
self.assertDictEqual(del_blk,
g.get_block_byid(self.foo_block['block_id']))
self.assertTrue(int(time.time()) - del_blk["mtime"] < 3)
fake_bid = BlockID('d0', 'g000640000000125', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1)
self.assertRaises(BlockNotFoundError, g.mark_delete_block_byid,
fake_bid)
def test_get_replica_blocks(self):
ec_blk_idxes = ['0000', '0001', '0003']
replica_blk_idxes = ['0002', '0008', '0012']
bg = self.make_test_block_group(ec_blk_idxes + replica_blk_idxes)
replica_blks = bg.indexes_to_blocks(replica_blk_idxes)
for blk in replica_blks:
bid = blk['block_id']
act_replica_blks = bg.get_replica_blocks(bid)
self.assertListEqual(replica_blks, act_replica_blks)
_replica_blks = copy.deepcopy(replica_blks)
_replica_blks.remove(blk)
act_replica_blks = bg.get_replica_blocks(bid, include_me=False)
self.assertListEqual(_replica_blks, act_replica_blks)
fake_bid = BlockID('d0', 'g000640000000125', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1)
self.assertIsNone(bg.get_replica_blocks(fake_bid, raise_error=False))
self.assertRaises(BlockNotFoundError,
bg.get_replica_blocks,
fake_bid,
raise_error=True)
self.assertRaises(BlockNotFoundError, bg.get_replica_blocks, fake_bid)
def setUp(self):
self.block_group_id = BlockGroupID('g000640000000123')
self.block_id = BlockID(
'd1g0006300000001230101idc000c62d8736c72800020000000001')
self.block_index = BlockIndex('1234')
self.drive_id = DriveID('idc000' '1122334455660001')
def make_test_block_group(self, blk_idxes, config=None):
gid = 'g000640000000123'
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
if config is None:
config = _ec_config
num_idcs = sum(config['cross_idc'])
idcs = ['idc' + (str(i).rjust(3, '0')) for i in range(num_idcs)]
bg = BlockGroup(block_group_id=gid, idcs=idcs, config=config)
for i, bi in enumerate(blk_idxes):
bi = BlockIndex(bi)
typ = bg.get_block_type(bi)
drive_id = DriveID(idcs[int(bi[0])] + 'c62d8736c7280002')
blkid = BlockID(typ, gid, bi, drive_id, i)
blk = copy.deepcopy(base_blk)
blk['block_id'] = blkid
bg.add_block(blk)
return bg
def test_is_ec_block(self):
idc_idx = 0
ec_blk_idxes = ['0000', '0001', '0005']
replica_blk_idxes = ['0002', '0008', '0012']
bg = self.make_test_block_group(ec_blk_idxes + replica_blk_idxes)
with self.assertRaises(BlockNotFoundError):
gid = 'g000640000000123'
bid = BlockID('dp', gid, '0001',
DriveID('idc000'
'ab2d8736c7280002'), 0)
bg.is_ec_block(bid)
act_ec_blk_idxes = []
nr_data, nr_parity = bg['config']['in_idc']
for i in range(0, nr_data + nr_parity):
bi = BlockIndex(idc_idx, i)
blk = bg.get_block(bi)
if blk is None:
continue
if bg.is_ec_block(blk['block_id']):
act_ec_blk_idxes.append(bi)
self.assertListEqual(ec_blk_idxes, act_ec_blk_idxes)
def test_json(self):
blk = BlockDesc({
'block_id':
BlockID('d0', 'g000640000000123', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1),
'size':
1000,
'range': ['0a', '0b'],
'ts_range': ["1235", "456"],
'ref_num':
1,
'is_del':
0,
'mtime':
1,
})
rst = utfjson.dump(blk)
expected = (
'{"block_id": "d0g0006400000001230000idc000c62d8736c72800020000000001", "is_del": 0, "ref_num": 1, "range": ["0a", "0b"], "mtime": 1, "ts_range": ["1235", "456"], "size": 1000}'
)
self.assertEqual(expected, rst)
loaded = BlockDesc(utfjson.load(rst))
self.assertEqual(blk, loaded)
def test_classify_blocks(self):
gid = 'g000640000000123'
g = BlockGroup(block_group_id=gid,
idcs=['a', 'b', 'c'],
config=_ec_config)
blks = g.classify_blocks(0, only_primary=True)
self.assertEqual([], blks['ec'] + blks['replica'] + blks['mark_del'])
base_blk = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
ec_blk_idxes = ['0000', '0001']
replica_blk_idxes = ['0002', '0008', '0012']
mark_del_idxes = ['0003', '0004']
for i, idx in enumerate(ec_blk_idxes + replica_blk_idxes +
mark_del_idxes):
typ = g.get_block_type(idx)
blkid = BlockID(typ, gid, idx, DriveID('idc000'
'c62d8736c7280002'), i)
blk = copy.deepcopy(base_blk)
blk['block_id'] = blkid
if idx in mark_del_idxes:
blk['is_del'] = 1
g.add_block(blk)
for only_primary in (True, False):
blks = g.classify_blocks(0, only_primary)
blk_idxes = []
for blk in blks['ec'] + blks['replica'] + blks['mark_del']:
idx = BlockID(blk['block_id']).block_index
blk_idxes.append(idx)
expect_ids = copy.deepcopy(ec_blk_idxes)
#'0004' in ec_blk_idxes is parity, so should not in mark_del
if only_primary is True:
expect_ids += replica_blk_idxes[:1] + mark_del_idxes[:1]
else:
expect_ids += replica_blk_idxes + mark_del_idxes[:1]
self.assertEqual(expect_ids, blk_idxes)
def setUp(self):
self.foo_block = BlockDesc({
'block_id':
BlockID('d0', 'g000640000000123', '0000',
DriveID('c62d8736c7280002'), 1),
'size':
1000,
'range': ['0a', '0b'],
'is_del':
0
})
def test_block_id_block_id(self):
block_id = 'd1g0006300000001230101idc000c62d8736c72800020000000001'
bid = BlockID('d1', 'g000630000000123', '0101',
DriveID('idc000'
'c62d8736c728'
'0002'), 1)
self.assertEqual(
'd1g0006300000001230101idc000c62d8736c72800020000000001',
bid.block_id)
self.assertEqual(block_id, bid.block_id)
self.assertIs(bid, bid.block_id)
def test_block_id_embed(self):
block_id = 'd1g0006300000001230101idc000c62d8736c72800020000000001'
bid = BlockID('d1', 'g000630000000123', '0101',
DriveID('idc000'
'c62d8736c728'
'0002'), 1)
# embedded drive_id attrs
self.assertEqual('idc000' 'c62d8736c728', bid.server_id)
self.assertEqual('002', bid.mountpoint_index)
self.assertEqual(6002, bid.port)
# embedded server_id attrs
self.assertEqual('idc000', bid.idc_id)
self.assertEqual('c62d8736c728', bid.mac_addr)
def test_json(self):
blk = BlockDesc({
'block_id': BlockID('d0', 'g000640000000123', '0000',
DriveID('c62d8736c7280002'), 1),
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
rst = utfjson.dump(blk)
expected = ('{"is_del": 0, "range": ["0a", "0b"], "block_id": '
'"d0g0006400000001230000c62d8736c72800020000000001", "size": 1000}')
self.assertEqual(expected, rst)
loaded = BlockDesc(utfjson.load(rst))
self.assertEqual(blk, loaded)
def test_new(self):
block_id = 'd1g0006300000001230101c62d8736c72800020000000001'
_bid = BlockID('d1', 'g000630000000123', '0101',
DriveID('c62d8736c7280002'), 1)
self.assertEqual(block_id, str(_bid))
self.assertEqual(_bid, BlockID(block_id))
self.assertEqual(_bid, BlockID(_bid))
bid = BlockID(block_id)
self.assertEqual('d1', bid.type)
self.assertEqual('g000630000000123', str(bid.block_group_id))
self.assertEqual((1, 1), bid.block_index.as_tuple())
self.assertEqual('0101', str(bid.block_index))
self.assertEqual('c62d8736c7280002', bid.drive_id)
self.assertEqual(1, bid.block_id_seq)
def test_get_parities(self):
gid = 'g000640000000123'
g = BlockGroup(block_group_id=gid,
idcs=['a', 'b', 'c'],
config=_ec_config)
parities = g.get_parities(idc_index=0)
self.assertEqual([], parities)
base_parity = BlockDesc({
'size': 1000,
'range': ['0a', '0b'],
'is_del': 0
})
parity_idxes = ['0004', '0005']
for i, idx in enumerate(parity_idxes):
blkid = BlockID('dp', gid, idx, DriveID('idc000'
'c62d8736c7280002'), i)
parity = copy.deepcopy(base_parity)
parity['block_id'] = blkid
g.add_block(parity)
idxes = g.get_parity_indexes(idc_index=0)
self.assertEqual(parity_idxes, idxes)
parities = g.get_parities(idc_index=0)
idxes = []
for p in parities:
idx = BlockID(p['block_id']).block_index
idxes.append(idx)
self.assertEqual(parity_idxes, idxes)
def test_get_block_idc(self):
g = BlockGroup(block_group_id='g000640000000123',
idcs=['a', 'b', 'c'],
config=_ec_config)
self.assertEqual('a', g.get_block_idc('0000'))
self.assertEqual('b', g.get_block_idc('0100'))
self.assertEqual('c', g.get_block_idc('0200'))
d0 = BlockDesc({
'block_id':
BlockID('d0', 'g000640000000123', '0000',
DriveID('c62d8736c7280002'), 1),
'size':
1000,
'range': ['0a', '0b'],
'is_del':
0
})
g.add_block(d0)
self.assertEqual('a', g.get_block_idc('0000'))
def test_get_block_byid(self):
blk_idxes = ['0000', '0001', '0002', '0003', '0008', '0012']
bg = self.make_test_block_group(blk_idxes)
blks = bg.indexes_to_blocks(blk_idxes)
bids = [blk['block_id'] for blk in blks]
act_blks = []
for bid in bids:
act_blks.append(bg.get_block_byid(bid))
self.assertListEqual(blks, act_blks)
fake_bid = BlockID('d0', 'g000640000000125', '0000',
DriveID('idc000'
'c62d8736c7280002'), 1)
self.assertIsNone(bg.get_block_byid(fake_bid, raise_error=False))
self.assertRaises(BlockNotFoundError, bg.get_block_byid, fake_bid,
True)
self.assertRaises(BlockNotFoundError, bg.get_block_byid, fake_bid)
def test_drive_id_embed(self):
d = DriveID('idc000' '112233445566' '0001')
self.assertEqual('idc000', d.idc_id)
self.assertEqual('112233445566', d.mac_addr)
def test_drive_id_self(self):
d = DriveID('idc0001122334455660001')
self.assertEqual('idc0001122334455660001', d.drive_id)
self.assertEqual(d, d.drive_id)
self.assertIs(d, d.drive_id)