def test_restart_on_inconsistency(looper, txnPoolNodeSet, tconf, tdir, monkeypatch):
restarted = multiprocessing.Value('i', 0)
def notify_restart():
with restarted.get_lock():
restarted.value = 1
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, 'teststdout')
monkeypatch.setattr(tool, '_restart', notify_restart)
def check_restart(value):
with restarted.get_lock():
assert restarted.value == value
nct = NodeControlToolExecutor(backup_dir=tdir, backup_target=tdir, transform=transform, config=tconf)
try:
# Trigger inconsistent 3PC state event
txnPoolNodeSet[0].on_inconsistent_3pc_state()
# Ensure we don't restart before timeout
looper.runFor(0.8 * RESTART_TIMEOUT)
check_restart(0)
# Ensure we restart eventually
looper.run(eventually(check_restart, 1, retryWait=1.0))
finally:
nct.stop()
def test_communication_with_node_control_tool(looper, tdir, tconf,
monkeypatch):
received = m.list()
msg = RESTART_MESSAGE
stdout = 'teststdout'
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_restart', restart_count)
def check_restart_count():
assert len(received) == 1
def restart_count():
received.append(RESTART_MESSAGE)
nct = NCT(backup_dir=tdir,
backup_target=tdir,
transform=transform,
config=tconf)
try:
send_restart_message(msg)
looper.run(eventually(check_restart_count))
finally:
nct.stop()
def testNodeControlCreatesBackups(monkeypatch, tdir, looper, tconf):
version = bumpedVersion()
stdout = 'teststdout'
curr_src_ver = Upgrader.get_src_version()
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_remove_old_backups', lambda *x: None)
def checkBackup(tool):
assert os.path.isfile('{}.{}'.format(
tool._backup_name(curr_src_ver.release), tool.backup_format))
monkeypatch.setattr(
NodeControlUtil, 'get_latest_pkg_version',
lambda *x, **y: DebianVersion(
version, upstream_cls=src_version_cls())
)
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform, config=tconf)
try:
sendUpgradeMessage(version)
looper.run(eventually(checkBackup, nct.tool))
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def test_upg_default_cfg(tdir, monkeypatch, tconf):
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, "")
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
assert nct.tool.ext_ver is None
assert nct.tool.deps == DEPS
assert nct.tool.packages_to_hold.strip(" ") == PACKAGES_TO_HOLD.strip(
" ")
finally:
nct.stop()
def testNodeControlRemovesBackups(monkeypatch, tdir, looper, tconf):
version = bumpedVersion()
stdout = 'teststdout'
curr_src_ver = Upgrader.get_src_version()
backupsWereRemoved = m.Value('b', False)
def testRemoveOldBackups(tool):
assert len(tool._get_backups()) == (tool.backup_num + 1)
#looper = Looper(debug=True)
tool._remove_old_backups_test()
backupsWereRemoved.value = True
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
tool._remove_old_backups_test = tool._remove_old_backups
monkeypatch.setattr(tool, '_remove_old_backups',
functools.partial(testRemoveOldBackups, tool))
def checkOldBackupsRemoved():
assert backupsWereRemoved.value
def check_backups_files_exists():
assert os.path.exists('{}.{}'.format(
nct.tool._backup_name(curr_src_ver.release),
nct.tool.backup_format))
monkeypatch.setattr(
NodeControlUtil, 'get_latest_pkg_version',
lambda *x, **y: DebianVersion(version, upstream_cls=src_version_cls()))
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
assert len(nct.tool._get_backups()) == 0
for i in range(nct.tool.backup_num):
file = os.path.join(nct.tool.base_dir,
'{}{}'.format(nct.tool.backup_name_prefix, i))
with open(file, 'w') as f:
f.write('random')
assert len(nct.tool._get_backups()) == nct.tool.backup_num
sendUpgradeMessage(version)
looper.run(eventually(checkOldBackupsRemoved))
looper.run(eventually(check_backups_files_exists))
assert len(nct.tool._get_backups()) == nct.tool.backup_num
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def testNodeControlPerformsMigrations(monkeypatch, tdir, looper, tconf):
version = bumpedVersion()
stdout = 'teststdout'
migrationFile = 'migrationProof'
migrationText = "{} {}".format(releaseVersion(), version)
old_call_upgrade_script = None
def mock_call_upgrade_script(*args, **kwargs):
old_call_upgrade_script(*args, **kwargs)
monkeypatch.setattr(
NodeControlUtil,
'_get_curr_info',
lambda *x, **y: (
"Package: {}\nVersion: {}\n".format(APP_NAME, version)
)
)
def mock_migrate(curr_src_ver: str, new_src_ver: str):
with open(os.path.join(tdir, migrationFile), 'w') as f:
f.write("{} {}".format(curr_src_ver, new_src_ver))
def transform(tool):
nonlocal old_call_upgrade_script
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_do_migration', mock_migrate)
old_call_upgrade_script = getattr(tool, '_call_upgrade_script')
monkeypatch.setattr(tool, '_call_upgrade_script', mock_call_upgrade_script)
def checkMigration():
with open(os.path.join(tdir, migrationFile)) as f:
assert f.read() == migrationText
monkeypatch.setattr(
NodeControlUtil, 'get_latest_pkg_version',
lambda *x, **y: DebianVersion(
version, upstream_cls=src_version_cls())
)
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform, config=tconf)
try:
sendUpgradeMessage(version)
looper.run(eventually(checkMigration))
finally:
nct.stop()
def testNodeControlRestoresFromBackups(monkeypatch, tdir, looper, tconf):
version = bumpedVersion()
stdout = 'teststdout'
backupWasRestored = m.Value('b', False)
testFile = 'testFile'
original_text = '1'
new_text = '2'
def testRestoreBackup(tool, src_ver: str):
tool._restore_from_backup_test(src_ver)
backupWasRestored.value = True
def mockMigrate(tool, *args):
monkeypatch.setattr(tool, '_do_migration', lambda *args: None)
with open(os.path.join(tool.indy_dir, testFile), 'w') as f:
f.write(new_text)
raise Exception('test')
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
tool._restore_from_backup_test = tool._restore_from_backup
monkeypatch.setattr(tool, '_do_migration',
functools.partial(mockMigrate, tool))
monkeypatch.setattr(tool, '_restore_from_backup',
functools.partial(testRestoreBackup, tool))
def checkBackupRestored(tool):
assert backupWasRestored.value
monkeypatch.setattr(
NodeControlUtil, 'get_latest_pkg_version',
lambda *x, **y: DebianVersion(version, upstream_cls=src_version_cls()))
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
with open(os.path.join(nct.tool.indy_dir, testFile), 'w') as f:
f.write(original_text)
sendUpgradeMessage(version)
looper.run(eventually(checkBackupRestored, nct.tool))
with open(os.path.join(nct.tool.indy_dir, testFile)) as f:
assert original_text == f.read()
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def testNodeControlReceivesMessages(monkeypatch, looper, tdir):
received = m.list()
msg = 'test'
stdout = 'teststdout'
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_process_data', received.append)
def checkMessage():
assert len(received) == 1
assert received[0] == composeUpgradeMessage(msg)
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
sendUpgradeMessage(msg)
looper.run(eventually(checkMessage))
finally:
nct.stop()
def test_upg_ext_info(tdir, monkeypatch, tconf):
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir,
PACKAGE_MNG_EXT_PTK_OUTPUT)
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
assert EXT_PKT_VERSION, EXT_PKT_DEPS == nct.tool._ext_info()
nct.tool._ext_init()
assert nct.tool.ext_ver == EXT_PKT_VERSION
assert nct.tool.deps == EXT_PKT_DEPS + DEPS
hlds = list(
set([EXT_PKT_NAME] + EXT_PKT_DEPS +
PACKAGES_TO_HOLD.strip(" ").split(" "))).sort()
cmp_with = list(set(
nct.tool.packages_to_hold.strip(" ").split(" "))).sort()
assert cmp_with == hlds
finally:
nct.stop()
def testNodeControlReceivesMessages(monkeypatch, looper, tdir):
received = m.list()
msg = 'test'
stdout = 'teststdout'
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_process_data', received.append)
def checkMessage():
assert len(received) == 1
assert received[0] == composeUpgradeMessage(msg)
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
sendUpgradeMessage(msg)
looper.run(eventually(checkMessage))
finally:
nct.stop()
def test_node_control_tool_restart(looper, tdir, monkeypatch, tconf):
received = m.list()
msg = RESTART_MESSAGE
stdout = 'teststdout'
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_process_data', received.append)
def check_message():
assert len(received) == 1
assert received[0] == compose_restart_message(msg)
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
send_restart_message(msg)
looper.run(eventually(check_message))
finally:
nct.stop()
def testNodeControlCreatesBackups(monkeypatch, tdir, looper, tconf):
msg = 'test'
stdout = 'teststdout'
currentVersion = Upgrader.getVersion()
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_remove_old_backups', lambda *x: None)
def checkBackup(tool):
assert os.path.isfile('{}.{}'.format(
tool._backup_name(currentVersion), tool.backup_format))
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
sendUpgradeMessage(msg)
looper.run(eventually(checkBackup, nct.tool))
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def testNodeControlCreatesBackups(monkeypatch, tdir, looper):
msg = 'test'
stdout = 'teststdout'
currentVersion = Upgrader.getVersion()
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_remove_old_backups', lambda *x: None)
def checkBackup(tool):
assert os.path.isfile('{}.{}'.format(
tool._backup_name(currentVersion), tool.backup_format))
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
sendUpgradeMessage(msg)
looper.run(eventually(checkBackup, nct.tool))
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def testNodeControlRestoresFromBackups(monkeypatch, tdir, looper, tconf):
msg = 'test'
stdout = 'teststdout'
currentVersion = Upgrader.getVersion()
backupWasRestored = m.Value('b', False)
testFile = 'testFile'
original_text = '1'
new_text = '2'
def testRestoreBackup(tool, version):
tool._restore_from_backup_test(version)
backupWasRestored.value = True
def mockMigrate(tool, *args):
monkeypatch.setattr(tool, '_migrate', lambda *args: None)
with open(os.path.join(tool.indy_dir, testFile), 'w') as f:
f.write(new_text)
raise Exception('test')
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
tool._restore_from_backup_test = tool._restore_from_backup
monkeypatch.setattr(tool, '_migrate',
functools.partial(mockMigrate, tool))
monkeypatch.setattr(tool, '_restore_from_backup',
functools.partial(testRestoreBackup, tool))
def checkBackupRestored(tool):
assert backupWasRestored.value
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
with open(os.path.join(nct.tool.indy_dir, testFile), 'w') as f:
f.write(original_text)
sendUpgradeMessage(msg)
looper.run(eventually(checkBackupRestored, nct.tool))
with open(os.path.join(nct.tool.indy_dir, testFile)) as f:
assert original_text == f.read()
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def testNodeControlRestoresFromBackups(monkeypatch, tdir, looper, tconf):
msg = 'test'
stdout = 'teststdout'
currentVersion = Upgrader.getVersion()
backupWasRestored = m.Value('b', False)
testFile = 'testFile'
original_text = '1'
new_text = '2'
def testRestoreBackup(tool, version):
tool._restore_from_backup_test(version)
backupWasRestored.value = True
def mockMigrate(tool, *args):
monkeypatch.setattr(tool, '_migrate', lambda *args: None)
with open(os.path.join(tool.indy_dir, testFile), 'w') as f:
f.write(new_text)
raise Exception('test')
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
tool._restore_from_backup_test = tool._restore_from_backup
monkeypatch.setattr(
tool, '_migrate', functools.partial(mockMigrate, tool))
monkeypatch.setattr(tool, '_restore_from_backup',
functools.partial(testRestoreBackup, tool))
def checkBackupRestored(tool):
assert backupWasRestored.value
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
with open(os.path.join(nct.tool.indy_dir, testFile), 'w') as f:
f.write(original_text)
sendUpgradeMessage(msg)
looper.run(eventually(checkBackupRestored, nct.tool))
with open(os.path.join(nct.tool.indy_dir, testFile)) as f:
assert original_text == f.read()
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def testNodeControlRemovesBackups(monkeypatch, tdir, looper):
msg = 'test'
stdout = 'teststdout'
currentVersion = Upgrader.getVersion()
backupsWereRemoved = m.Value('b', False)
def testRemoveOldBackups(tool):
assert len(tool._get_backups()) == (tool.backup_num + 1)
#looper = Looper(debug=True)
tool._remove_old_backups_test()
backupsWereRemoved.value = True
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
tool._remove_old_backups_test = tool._remove_old_backups
monkeypatch.setattr(tool, '_remove_old_backups',
functools.partial(testRemoveOldBackups, tool))
def checkOldBackupsRemoved():
assert backupsWereRemoved.value
def check_backups_files_exists():
assert os.path.exists('{}.{}'.format(
nct.tool._backup_name(currentVersion), nct.tool.backup_format))
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
assert len(nct.tool._get_backups()) == 0
for i in range(nct.tool.backup_num):
file = os.path.join(nct.tool.base_dir, '{}{}'.format(
nct.tool.backup_name_prefix, i))
with open(file, 'w') as f:
f.write('random')
assert len(nct.tool._get_backups()) == nct.tool.backup_num
sendUpgradeMessage(msg)
looper.run(eventually(checkOldBackupsRemoved))
looper.run(eventually(check_backups_files_exists))
assert len(nct.tool._get_backups()) == nct.tool.backup_num
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def test_communication_with_node_control_tool(looper, tdir, tconf, monkeypatch):
received = m.list()
msg = RESTART_MESSAGE
stdout = 'teststdout'
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_restart', restart_count)
def check_restart_count():
assert len(received) == 1
def restart_count():
received.append(RESTART_MESSAGE)
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
send_restart_message(msg)
looper.run(eventually(check_restart_count))
finally:
nct.stop()
def test_node_control_tool_restart(looper, tdir, monkeypatch, tconf):
received = m.list()
msg = RESTART_MESSAGE
stdout = 'teststdout'
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_process_data', received.append)
def check_message():
assert len(received) == 1
assert received[0] == compose_restart_message(msg)
nct = NCT(backup_dir=tdir,
backup_target=tdir,
transform=transform,
config=tconf)
try:
send_restart_message(msg)
looper.run(eventually(check_message))
finally:
nct.stop()
def test_restart_on_inconsistency(looper, txnPoolNodeSet, tdir, monkeypatch):
restarted = multiprocessing.Value('i', 0)
def notify_restart():
with restarted.get_lock():
restarted.value = 1
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, 'teststdout')
monkeypatch.setattr(tool, '_restart', notify_restart)
def check_restart():
with restarted.get_lock():
assert restarted.value == 1
nct = NodeControlToolExecutor(backup_dir=tdir,
backup_target=tdir,
transform=transform)
try:
txnPoolNodeSet[0].on_inconsistent_3pc_state()
looper.run(eventually(check_restart))
finally:
nct.stop()
def testNodeControlRemovesBackups(monkeypatch, tdir, looper):
msg = 'test'
stdout = 'teststdout'
currentVersion = Upgrader.getVersion()
backupsWereRemoved = m.Value('b', False)
def testRemoveOldBackups(tool):
assert len(tool._get_backups()) == (tool.backup_num + 1)
tool._remove_old_backups_test()
backupsWereRemoved.value = True
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
tool._remove_old_backups_test = tool._remove_old_backups
monkeypatch.setattr(tool, '_remove_old_backups',
functools.partial(testRemoveOldBackups, tool))
def checkOldBackupsRemoved():
assert backupsWereRemoved.value
nct = NCT(transform=transform)
try:
assert len(nct.tool._get_backups()) == 0
for i in range(nct.tool.backup_num):
file = os.path.join(nct.tool.base_dir,
'{}{}'.format(nct.tool.backup_name_prefix, i))
with open(file, 'w') as f:
f.write('random')
assert len(nct.tool._get_backups()) == nct.tool.backup_num
sendUpgradeMessage(msg)
looper.run(eventually(checkOldBackupsRemoved))
assert os.path.exists('{}.{}'.format(
nct.tool._backup_name(currentVersion), nct.tool.backup_format))
assert len(nct.tool._get_backups()) == nct.tool.backup_num
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def testNodeControlPerformsMigrations(monkeypatch, tdir, looper):
msg = 'test'
stdout = 'teststdout'
migrationFile = 'migrationProof'
migrationText = 'testMigrations'
def mock_migrate(*args):
with open(os.path.join(tdir, migrationFile), 'w') as f:
f.write(migrationText)
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_migrate', mock_migrate)
def checkMigration():
with open(os.path.join(tdir, migrationFile)) as f:
assert f.read() == migrationText
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
sendUpgradeMessage(msg)
looper.run(eventually(checkMigration))
finally:
nct.stop()
def testNodeControlPerformsMigrations(monkeypatch, tdir, looper, tconf):
msg = 'test'
stdout = 'teststdout'
migrationFile = 'migrationProof'
migrationText = 'testMigrations'
def mock_migrate(*args):
with open(os.path.join(tdir, migrationFile), 'w') as f:
f.write(migrationText)
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_migrate', mock_migrate)
def checkMigration():
with open(os.path.join(tdir, migrationFile)) as f:
assert f.read() == migrationText
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
sendUpgradeMessage(msg)
looper.run(eventually(checkMigration))
finally:
nct.stop()