def test_hook_pre_post_device_locking(self):
locks = [None]
device_path = os.path.join(self.devices, self.existing_device)
datadir = 'object'
lock_file = os.path.join(device_path, '.relink.%s.lock' % datadir)
# The first run gets the lock
relinker.hook_pre_device(locks, {}, datadir, device_path)
self.assertNotEqual([None], locks)
# A following run would block
with self.assertRaises(IOError) as raised:
with open(lock_file, 'a') as f:
fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
self.assertEqual(errno.EAGAIN, raised.exception.errno)
# Another must not get the lock, so it must return an empty list
relinker.hook_post_device(locks, "")
self.assertEqual([None], locks)
with open(lock_file, 'a') as f:
fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
def test_state_file(self):
device_path = os.path.join(self.devices, self.existing_device)
datadir = 'objects'
datadir_path = os.path.join(device_path, datadir)
state_file = os.path.join(device_path, 'relink.%s.json' % datadir)
def call_partition_filter(part_power, next_part_power, parts):
# Partition 312 will be ignored because it must have been created
# by the relinker
return relinker.partitions_filter(states, part_power,
next_part_power, datadir_path,
parts)
# Start relinking
states = {
"part_power": PART_POWER,
"next_part_power": PART_POWER + 1,
"state": {}
}
# Load the states: As it starts, it must be empty
locks = [None]
relinker.hook_pre_device(locks, states, datadir, device_path)
self.assertEqual({}, states["state"])
os.close(locks[0]) # Release the lock
# Partition 312 is ignored because it must have been created with the
# next_part_power, so it does not need to be relinked
# 96 and 227 are reverse ordered
# auditor_status_ALL.json is ignored because it's not a partition
self.assertEqual(
['227', '96'],
call_partition_filter(PART_POWER, PART_POWER + 1,
['96', '227', '312', 'auditor_status.json']))
self.assertEqual(states["state"], {'96': False, '227': False})
pol = POLICIES[0]
mgr = DiskFileRouter({
'devices': self.devices,
'mount_check': False
}, self.logger)[pol]
# Ack partition 96
relinker.hook_post_partition(states, relinker.STEP_RELINK, pol, mgr,
os.path.join(datadir_path, '96'))
self.assertEqual(states["state"], {'96': True, '227': False})
with open(state_file, 'rt') as f:
self.assertEqual(
json.load(f), {
"part_power": PART_POWER,
"next_part_power": PART_POWER + 1,
"state": {
'96': True,
'227': False
}
})
# Restart relinking after only part 96 was done
self.assertEqual(['227'],
call_partition_filter(PART_POWER, PART_POWER + 1,
['96', '227', '312']))
self.assertEqual(states["state"], {'96': True, '227': False})
# Ack partition 227
relinker.hook_post_partition(states, relinker.STEP_RELINK, pol, mgr,
os.path.join(datadir_path, '227'))
self.assertEqual(states["state"], {'96': True, '227': True})
with open(state_file, 'rt') as f:
self.assertEqual(
json.load(f), {
"part_power": PART_POWER,
"next_part_power": PART_POWER + 1,
"state": {
'96': True,
'227': True
}
})
# If the process restarts, it reload the state
locks = [None]
states = {
"part_power": PART_POWER,
"next_part_power": PART_POWER + 1,
"state": {},
}
relinker.hook_pre_device(locks, states, datadir, device_path)
self.assertEqual(
states, {
"part_power": PART_POWER,
"next_part_power": PART_POWER + 1,
"state": {
'96': True,
'227': True
}
})
os.close(locks[0]) # Release the lock
# Start cleanup -- note that part_power and next_part_power now match!
states = {
"part_power": PART_POWER + 1,
"next_part_power": PART_POWER + 1,
"state": {},
}
# ...which means our state file was ignored
relinker.hook_pre_device(locks, states, datadir, device_path)
self.assertEqual(
states, {
"prev_part_power": PART_POWER,
"part_power": PART_POWER + 1,
"next_part_power": PART_POWER + 1,
"state": {}
})
os.close(locks[0]) # Release the lock
self.assertEqual(['227', '96'],
call_partition_filter(PART_POWER + 1, PART_POWER + 1,
['96', '227', '312']))
# Ack partition 227
relinker.hook_post_partition(states, relinker.STEP_CLEANUP, pol, mgr,
os.path.join(datadir_path, '227'))
self.assertEqual(states["state"], {'96': False, '227': True})
with open(state_file, 'rt') as f:
self.assertEqual(
json.load(f), {
"prev_part_power": PART_POWER,
"part_power": PART_POWER + 1,
"next_part_power": PART_POWER + 1,
"state": {
'96': False,
'227': True
}
})
# Restart cleanup after only part 227 was done
self.assertEqual(['96'],
call_partition_filter(PART_POWER + 1, PART_POWER + 1,
['96', '227', '312']))
self.assertEqual(states["state"], {'96': False, '227': True})
# Ack partition 96
relinker.hook_post_partition(states, relinker.STEP_CLEANUP, pol, mgr,
os.path.join(datadir_path, '96'))
self.assertEqual(states["state"], {'96': True, '227': True})
with open(state_file, 'rt') as f:
self.assertEqual(
json.load(f), {
"prev_part_power": PART_POWER,
"part_power": PART_POWER + 1,
"next_part_power": PART_POWER + 1,
"state": {
'96': True,
'227': True
}
})
# At the end, the state is still accurate
locks = [None]
states = {
"prev_part_power": PART_POWER,
"part_power": PART_POWER + 1,
"next_part_power": PART_POWER + 1,
"state": {},
}
relinker.hook_pre_device(locks, states, datadir, device_path)
self.assertEqual(states["state"], {'96': True, '227': True})
os.close(locks[0]) # Release the lock
# If the part_power/next_part_power tuple differs, restart from scratch
locks = [None]
states = {
"part_power": PART_POWER + 1,
"next_part_power": PART_POWER + 2,
"state": {},
}
relinker.hook_pre_device(locks, states, datadir, device_path)
self.assertEqual(states["state"], {})
self.assertFalse(os.path.exists(state_file))
os.close(locks[0]) # Release the lock
# If the file gets corrupted, restart from scratch
with open(state_file, 'wt') as f:
f.write('NOT JSON')
locks = [None]
states = {
"part_power": PART_POWER,
"next_part_power": PART_POWER + 1,
"state": {}
}
relinker.hook_pre_device(locks, states, datadir, device_path)
self.assertEqual(states["state"], {})
self.assertFalse(os.path.exists(state_file))
os.close(locks[0]) # Release the lock
def test_state_file(self):
device_path = os.path.join(self.devices, self.existing_device)
datadir = 'objects'
datadir_path = os.path.join(device_path, datadir)
state_file = os.path.join(device_path, 'relink.%s.json' % datadir)
def call_partition_filter(step, parts):
# Partition 312 will be ignored because it must have been created
# by the relinker
return relinker.partitions_filter(states, step,
PART_POWER, PART_POWER + 1,
datadir_path, parts)
# Start relinking
states = {}
# Load the states: As it starts, it must be empty
locks = [None]
relinker.hook_pre_device(locks, states, datadir, device_path)
self.assertEqual({}, states)
os.close(locks[0]) # Release the lock
# Partition 312 is ignored because it must have been created with the
# next_part_power, so it does not need to be relinked
# 96 and 227 are reverse ordered
# auditor_status_ALL.json is ignored because it's not a partition
self.assertEqual(['227', '96'],
call_partition_filter(relinker.STEP_RELINK,
['96', '227', '312',
'auditor_status.json']))
self.assertEqual(states, {'96': [False, False], '227': [False, False]})
# Ack partition 96
relinker.hook_post_partition(states, relinker.STEP_RELINK,
os.path.join(datadir_path, '96'))
self.assertEqual(states, {'96': [True, False], '227': [False, False]})
with open(state_file, 'rt') as f:
self.assertEqual(json.load(f), {'96': [True, False],
'227': [False, False]})
# Restart relinking after only part 96 was done
self.assertEqual(['227'],
call_partition_filter(relinker.STEP_RELINK,
['96', '227', '312']))
self.assertEqual(states, {'96': [True, False], '227': [False, False]})
# Ack partition 227
relinker.hook_post_partition(states, relinker.STEP_RELINK,
os.path.join(datadir_path, '227'))
self.assertEqual(states, {'96': [True, False], '227': [True, False]})
with open(state_file, 'rt') as f:
self.assertEqual(json.load(f), {'96': [True, False],
'227': [True, False]})
# If the process restarts, it reload the state
locks = [None]
states = {}
relinker.hook_pre_device(locks, states, datadir, device_path)
self.assertEqual(states, {'96': [True, False], '227': [True, False]})
os.close(locks[0]) # Release the lock
# Start cleanup
self.assertEqual(['227', '96'],
call_partition_filter(relinker.STEP_CLEANUP,
['96', '227', '312']))
# Ack partition 227
relinker.hook_post_partition(states, relinker.STEP_CLEANUP,
os.path.join(datadir_path, '227'))
self.assertEqual(states, {'96': [True, False], '227': [True, True]})
with open(state_file, 'rt') as f:
self.assertEqual(json.load(f), {'96': [True, False],
'227': [True, True]})
# Restart cleanup after only part 227 was done
self.assertEqual(['96'],
call_partition_filter(relinker.STEP_CLEANUP,
['96', '227', '312']))
self.assertEqual(states, {'96': [True, False], '227': [True, True]})
# Ack partition 96
relinker.hook_post_partition(states, relinker.STEP_CLEANUP,
os.path.join(datadir_path, '96'))
self.assertEqual(states, {'96': [True, True], '227': [True, True]})
with open(state_file, 'rt') as f:
self.assertEqual(json.load(f), {'96': [True, True],
'227': [True, True]})
# At the end, the state is still accurate
locks = [None]
states = {}
relinker.hook_pre_device(locks, states, datadir, device_path)
self.assertEqual(states, {'96': [True, True], '227': [True, True]})
os.close(locks[0]) # Release the lock
# If the file gets corrupted, restart from scratch
with open(state_file, 'wt') as f:
f.write('NOT JSON')
locks = [None]
states = {}
relinker.hook_pre_device(locks, states, datadir, device_path)
self.assertEqual(states, {})
os.close(locks[0]) # Release the lock