def test_upgrade_does_not_get_into_loop_if_reinstall(
looper,
tconf,
nodeSet,
validUpgrade,
sdk_pool_handle,
sdk_wallet_trustee,
monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
upgr1[REINSTALL] = True
# An upgrade scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# here we make nodes think they have upgraded successfully
monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version)
check_no_loop(nodeSet, UpgradeLog.SUCCEEDED)
def testScheduleNodeUpgrade(tconf, nodeSet):
"""
Tests that upgrade scheduling works. For that it starts mock
control service, schedules upgrade for near future and then checks that
service received notification.
"""
loop = asyncio.get_event_loop()
server, indicator = loop.run_until_complete(
_createServer(
host=tconf.controlServiceHost,
port=tconf.controlServicePort
)
)
indicator.add_done_callback(_stopServer(server))
node = nodeSet[0]
# ATTENTION! nodeId and ledger must not be None, but there
# we do not call methods that use them, so we can pass None
# We do it because node from nodeSet is some testable object, not real
# node, so it has no nodeId and ledger that we can use
upgrader = Upgrader(nodeId=None,
nodeName=None,
dataDir=node.dataLocation,
config=tconf,
ledger=None)
version = bumpedVersion()
ev_data = UpgradeLogData(
datetime.utcnow(), version, 'some_id', tconf.UPGRADE_ENTRY)
upgrader._callUpgradeAgent(ev_data, 1000)
result = loop.run_until_complete(eventuallySoon(_checkFuture(indicator)))
expectedResult = UpgradeMessage(version, tconf.UPGRADE_ENTRY)
assert result == expectedResult.toJson()
def testOnlyTrusteeCanSendPoolUpgrade(looper, sdk_pool_handle, sdk_wallet_steward, validUpgrade):
# A steward sending POOL_UPGRADE but txn fails
validUpgrade = deepcopy(validUpgrade)
validUpgrade[NAME] = 'upgrade-20'
validUpgrade[VERSION] = bumpedVersion()
req = sdk_send_upgrade(looper, sdk_pool_handle, sdk_wallet_steward, validUpgrade)
sdk_get_bad_response(looper, [req], RequestRejectedException, 'cannot do')
def test_upgrade_does_not_get_into_loop_if_reinstall(looper, tconf, nodeSet,
validUpgrade,
sdk_pool_handle,
sdk_wallet_trustee,
monkeypatch):
new_version = bumpedVersion(validUpgrade['version'])
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
upgr1[REINSTALL] = True
clear_aq_stash(nodeSet)
# An upgrade scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# here we make nodes think they have upgraded successfully
monkeypatch.setattr(NodeControlUtil, '_get_curr_info',
lambda *x: "Version: {}".format(new_version))
check_no_loop(nodeSet, UpgradeLog.Events.succeeded)
def testNodeControlCreatesBackups(monkeypatch, tdir, looper, tconf):
version = bumpedVersion()
stdout = 'teststdout'
curr_src_ver = Upgrader.get_src_version()
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_remove_old_backups', lambda *x: None)
def checkBackup(tool):
assert os.path.isfile('{}.{}'.format(
tool._backup_name(curr_src_ver.release), tool.backup_format))
monkeypatch.setattr(
NodeControlUtil, 'get_latest_pkg_version',
lambda *x, **y: DebianVersion(
version, upstream_cls=src_version_cls())
)
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform, config=tconf)
try:
sendUpgradeMessage(version)
looper.run(eventually(checkBackup, nct.tool))
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def testOnlyTrusteeCanSendPoolUpgrade(looper, sdk_pool_handle,
sdk_wallet_steward, validUpgrade):
# A steward sending POOL_UPGRADE but txn fails
validUpgrade = deepcopy(validUpgrade)
validUpgrade[NAME] = 'upgrade-20'
validUpgrade[VERSION] = bumpedVersion()
req = sdk_send_upgrade(looper, sdk_pool_handle, sdk_wallet_steward,
validUpgrade)
sdk_get_bad_response(looper, [req], RequestRejectedException, 'cannot do')
def testOnlyTrusteeCanSendPoolUpgrade(looper, steward, validUpgrade):
# A steward sending POOL_UPGRADE but txn fails
stClient, stWallet = steward
validUpgrade = deepcopy(validUpgrade)
validUpgrade[NAME] = 'upgrade-20'
validUpgrade[VERSION] = bumpedVersion()
_, req = sendUpgrade(stClient, stWallet, validUpgrade)
timeout = plenumWaits.expectedReqNAckQuorumTime()
looper.run(eventually(checkRejects, stClient, req.reqId,
'cannot do', retryWait=1, timeout=timeout))
def testOnlyTrusteeCanSendPoolUpgrade(looper, steward, validUpgrade):
# A steward sending POOL_UPGRADE but txn fails
stClient, stWallet = steward
validUpgrade = deepcopy(validUpgrade)
validUpgrade[NAME] = 'upgrade-20'
validUpgrade[VERSION] = bumpedVersion()
_, req = sendUpgrade(stClient, stWallet, validUpgrade)
timeout = plenumWaits.expectedReqNAckQuorumTime()
looper.run(eventually(checkRejects, stClient, req.reqId,
'cannot do', retryWait=1, timeout=timeout))
def invalidUpgrade(nodeIds, tconf):
schedule = {}
unow = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())
startAt = unow + timedelta(seconds=60)
acceptableDiff = tconf.MinSepBetweenNodeUpgrades + 1
for i in nodeIds:
schedule[i] = datetime.isoformat(startAt)
startAt = startAt + timedelta(seconds=acceptableDiff - 3)
return dict(name='upgrade-14', version=bumpedVersion(), action=START,
schedule=schedule,
# sha256=get_valid_code_hash(),
sha256='46c715a90b1067142d548cb1f1405b0486b32b1a27d418ef3a52bd976e9fae50',
timeout=10)
def validUpgrade(nodeIds, tconf):
schedule = {}
unow = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())
startAt = unow + timedelta(seconds=100)
acceptableDiff = tconf.MinSepBetweenNodeUpgrades + 1
for i in nodeIds:
schedule[i] = datetime.isoformat(startAt)
startAt = startAt + timedelta(seconds=acceptableDiff + 3)
return dict(name='upgrade-13', version=bumpedVersion(), action=START,
schedule=schedule,
# sha256=get_valid_code_hash(),
sha256='db34a72a90d026dae49c3b3f0436c8d3963476c77468ad955845a1ccf7b03f55',
timeout=1)
def invalidUpgrade(nodeIds, tconf):
schedule = {}
unow = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())
startAt = unow + timedelta(seconds=60)
acceptableDiff = tconf.MinSepBetweenNodeUpgrades + 1
for i in nodeIds:
schedule[i] = datetime.isoformat(startAt)
startAt = startAt + timedelta(seconds=acceptableDiff - 3)
return dict(name='upgrade-14', version=bumpedVersion(), action=START,
schedule=schedule,
# sha256=get_valid_code_hash(),
sha256='46c715a90b1067142d548cb1f1405b0486b32b1a27d418ef3a52bd976e9fae50',
timeout=10)
def validUpgrade(nodeIds, tconf):
schedule = {}
unow = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())
startAt = unow + timedelta(seconds=100)
acceptableDiff = tconf.MinSepBetweenNodeUpgrades + 1
for i in nodeIds:
schedule[i] = datetime.isoformat(startAt)
startAt = startAt + timedelta(seconds=acceptableDiff + 3)
return dict(name='upgrade-13', version=bumpedVersion(), action=START,
schedule=schedule,
# sha256=get_valid_code_hash(),
sha256='db34a72a90d026dae49c3b3f0436c8d3963476c77468ad955845a1ccf7b03f55',
timeout=1)
def validUpgrade(nodeIds, tconf, monkeypatch, pckg):
schedule = {}
unow = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())
startAt = unow + timedelta(seconds=100)
acceptableDiff = tconf.MinSepBetweenNodeUpgrades + 1
for i in nodeIds:
schedule[i] = datetime.isoformat(startAt)
startAt = startAt + timedelta(seconds=acceptableDiff + 3)
patch_packet_mgr_output(monkeypatch, pckg[0], pckg[1])
return dict(name='upgrade-{}'.format(randomText(3)), version=bumpedVersion(pckg[1]),
action=START, schedule=schedule, timeout=1, package=pckg[0],
sha256='db34a72a90d026dae49c3b3f0436c8d3963476c77468ad955845a1ccf7b03f55')
def testNodeControlRemovesBackups(monkeypatch, tdir, looper, tconf):
version = bumpedVersion()
stdout = 'teststdout'
curr_src_ver = Upgrader.get_src_version()
backupsWereRemoved = m.Value('b', False)
def testRemoveOldBackups(tool):
assert len(tool._get_backups()) == (tool.backup_num + 1)
#looper = Looper(debug=True)
tool._remove_old_backups_test()
backupsWereRemoved.value = True
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
tool._remove_old_backups_test = tool._remove_old_backups
monkeypatch.setattr(tool, '_remove_old_backups',
functools.partial(testRemoveOldBackups, tool))
def checkOldBackupsRemoved():
assert backupsWereRemoved.value
def check_backups_files_exists():
assert os.path.exists('{}.{}'.format(
nct.tool._backup_name(curr_src_ver.release),
nct.tool.backup_format))
monkeypatch.setattr(
NodeControlUtil, 'get_latest_pkg_version',
lambda *x, **y: DebianVersion(version, upstream_cls=src_version_cls()))
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
assert len(nct.tool._get_backups()) == 0
for i in range(nct.tool.backup_num):
file = os.path.join(nct.tool.base_dir,
'{}{}'.format(nct.tool.backup_name_prefix, i))
with open(file, 'w') as f:
f.write('random')
assert len(nct.tool._get_backups()) == nct.tool.backup_num
sendUpgradeMessage(version)
looper.run(eventually(checkOldBackupsRemoved))
looper.run(eventually(check_backups_files_exists))
assert len(nct.tool._get_backups()) == nct.tool.backup_num
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def testNodeControlPerformsMigrations(monkeypatch, tdir, looper, tconf):
version = bumpedVersion()
stdout = 'teststdout'
migrationFile = 'migrationProof'
migrationText = "{} {}".format(releaseVersion(), version)
old_call_upgrade_script = None
def mock_call_upgrade_script(*args, **kwargs):
old_call_upgrade_script(*args, **kwargs)
monkeypatch.setattr(
NodeControlUtil,
'_get_curr_info',
lambda *x, **y: (
"Package: {}\nVersion: {}\n".format(APP_NAME, version)
)
)
def mock_migrate(curr_src_ver: str, new_src_ver: str):
with open(os.path.join(tdir, migrationFile), 'w') as f:
f.write("{} {}".format(curr_src_ver, new_src_ver))
def transform(tool):
nonlocal old_call_upgrade_script
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
monkeypatch.setattr(tool, '_do_migration', mock_migrate)
old_call_upgrade_script = getattr(tool, '_call_upgrade_script')
monkeypatch.setattr(tool, '_call_upgrade_script', mock_call_upgrade_script)
def checkMigration():
with open(os.path.join(tdir, migrationFile)) as f:
assert f.read() == migrationText
monkeypatch.setattr(
NodeControlUtil, 'get_latest_pkg_version',
lambda *x, **y: DebianVersion(
version, upstream_cls=src_version_cls())
)
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform, config=tconf)
try:
sendUpgradeMessage(version)
looper.run(eventually(checkMigration))
finally:
nct.stop()
def test_upgrade_does_not_get_into_loop(looper, tconf, nodeSet, validUpgrade,
trustee, trusteeWallet, monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
# An upgrade scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# here we make nodes think they have upgraded successfully
monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version)
check_no_loop(nodeSet, UpgradeLog.SUCCEEDED)
def testNodeControlRestoresFromBackups(monkeypatch, tdir, looper, tconf):
version = bumpedVersion()
stdout = 'teststdout'
backupWasRestored = m.Value('b', False)
testFile = 'testFile'
original_text = '1'
new_text = '2'
def testRestoreBackup(tool, src_ver: str):
tool._restore_from_backup_test(src_ver)
backupWasRestored.value = True
def mockMigrate(tool, *args):
monkeypatch.setattr(tool, '_do_migration', lambda *args: None)
with open(os.path.join(tool.indy_dir, testFile), 'w') as f:
f.write(new_text)
raise Exception('test')
def transform(tool):
nodeControlGeneralMonkeypatching(tool, monkeypatch, tdir, stdout)
tool._restore_from_backup_test = tool._restore_from_backup
monkeypatch.setattr(tool, '_do_migration',
functools.partial(mockMigrate, tool))
monkeypatch.setattr(tool, '_restore_from_backup',
functools.partial(testRestoreBackup, tool))
def checkBackupRestored(tool):
assert backupWasRestored.value
monkeypatch.setattr(
NodeControlUtil, 'get_latest_pkg_version',
lambda *x, **y: DebianVersion(version, upstream_cls=src_version_cls()))
nct = NCT(backup_dir=tdir, backup_target=tdir, transform=transform)
try:
with open(os.path.join(nct.tool.indy_dir, testFile), 'w') as f:
f.write(original_text)
sendUpgradeMessage(version)
looper.run(eventually(checkBackupRestored, nct.tool))
with open(os.path.join(nct.tool.indy_dir, testFile)) as f:
assert original_text == f.read()
finally:
clean_dir(nct.tool.base_dir)
nct.stop()
def test_upgrade_does_not_get_into_loop_if_failed(looper, tconf, nodeSet,
validUpgrade, trustee,
trusteeWallet, monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
# An upgrade scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# we have not patched indy_node version so nodes think the upgrade had
# failed
check_no_loop(nodeSet, UpgradeLog.FAILED)
def test_upgrade_does_not_get_into_loop_if_failed(looper, tconf, nodeSet,
validUpgrade, sdk_pool_handle,
sdk_wallet_trustee, monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
# An upgrade scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# we have not patched indy_node version so nodes think the upgrade had
# failed
check_no_loop(nodeSet, UpgradeLog.FAILED)
def test_upgrade_does_not_get_into_loop(looper, tconf, nodeSet,
validUpgrade, trustee,
trusteeWallet, monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
# An upgrade scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# here we make nodes think they have upgraded successfully
monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version)
check_no_loop(nodeSet, UpgradeLog.UPGRADE_SUCCEEDED)
def test_upgrade_does_not_get_into_loop_force_if_failed(
looper, tconf, nodeSet, validUpgrade, trustee, trusteeWallet, monkeypatch):
new_version = bumpedVersion()
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
upgr1[FORCE] = True
# An upgrade scheduled, it should pass
ensureUpgradeSent(looper, trustee, trusteeWallet, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# we have not patched indy_node version so nodes think the upgrade had
# failed
check_no_loop(nodeSet, UpgradeLog.UPGRADE_FAILED)
def test_upgrade_does_not_get_into_loop_if_reinstall_and_failed(
looper, tconf, nodeSet, validUpgrade, sdk_pool_handle,
sdk_wallet_trustee, monkeypatch):
new_version = bumpedVersion(validUpgrade['version'])
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
upgr1[REINSTALL] = True
clear_aq_stash(nodeSet)
# An upgrade scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# we have not patched indy_node version so nodes think the upgrade had
# failed
check_no_loop(nodeSet, UpgradeLog.Events.failed)
def test_upgrade_does_not_get_into_loop_force(looper, tconf, nodeSet,
validUpgrade, sdk_pool_handle,
sdk_wallet_trustee, monkeypatch):
new_version = bumpedVersion(validUpgrade['version'])
upgr1 = deepcopy(validUpgrade)
upgr1[VERSION] = new_version
upgr1[FORCE] = True
clear_aq_stash(nodeSet)
# An upgrade scheduled, it should pass
sdk_ensure_upgrade_sent(looper, sdk_pool_handle, sdk_wallet_trustee, upgr1)
looper.run(
eventually(
checkUpgradeScheduled,
nodeSet,
upgr1[VERSION],
retryWait=1,
timeout=waits.expectedUpgradeScheduled()))
# here we make nodes think they have upgraded successfully
monkeypatch.setattr(indy_node.__metadata__, '__version__', new_version)
check_no_loop(nodeSet, UpgradeLog.SUCCEEDED)
import pytest
from indy_common.constants import FAIL
from indy_node.test.upgrade.helper import populate_log_with_upgrade_events, \
bumpedVersion, check_node_sent_acknowledges_upgrade, check_node_do_not_sent_acknowledges_upgrade, \
emulate_restart_pool_for_upgrade
from indy_node.server.upgrade_log import UpgradeLog
INVALID_VERSION = bumpedVersion()
whitelist = ['unable to send message',
'failed upgrade',
'This problem may have external reasons, check syslog for more information']
# TODO: Implement a client in node
@pytest.fixture(scope="module")
def tdirWithPoolTxns(tdirWithPoolTxns, poolTxnNodeNames, tdir, tconf):
# For each node, adding a file with he current version number which makes the node
# think that an upgrade has been performed
populate_log_with_upgrade_events(
poolTxnNodeNames, tdir, tconf, INVALID_VERSION)
return tdirWithPoolTxns
def test_node_detected_upgrade_failed(nodeSet):
'''
Test that each node checks Upgrade Log on startup (after Upgrade restart), and writes FAIL to it
because the current version differs from the one in Upgrade log.
Upgrade log already created START event (see fixture above emulating real upgrade)
import pytest
from indy_common.constants import IN_PROGRESS, FAIL
from indy_node.test.upgrade.helper import populate_log_with_upgrade_events, check_node_set_acknowledges_upgrade, \
bumpedVersion
from indy_node.server.upgrade_log import UpgradeLog
INVALID_VERSION = bumpedVersion()
whitelist = ['unable to send message', 'Failed to upgrade node']
# TODO: Implement a client in node
@pytest.fixture(scope="module")
def tdirWithPoolTxns(tdirWithPoolTxns, poolTxnNodeNames, tconf):
# For each node, adding a file with he current version number which makes the node
# think that an upgrade has been performed
populate_log_with_upgrade_events(tdirWithPoolTxns, poolTxnNodeNames, tconf,
INVALID_VERSION)
return tdirWithPoolTxns
def test_node_handles_unsuccessful_upgrade(looper, nodeSet, nodeIds):
check_node_set_acknowledges_upgrade(looper, nodeSet, nodeIds,
[IN_PROGRESS, FAIL], INVALID_VERSION)
for node in nodeSet:
assert node.upgrader.scheduledUpgrade is None
assert node.upgrader.lastUpgradeEventInfo[
0] == UpgradeLog.UPGRADE_FAILED
