def testScheduleNodeUpgrade(tconf, nodeSet):
"""
Tests that upgrade scheduling works. For that it starts mock
control service, schedules upgrade for near future and then checks that
service received notification.
"""
loop = asyncio.get_event_loop()
server, indicator = loop.run_until_complete(
_createServer(
host=tconf.controlServiceHost,
port=tconf.controlServicePort
)
)
indicator.add_done_callback(_stopServer(server))
node = nodeSet[0]
# ATTENTION! nodeId and ledger must not be None, but there
# we do not call methods that use them, so we can pass None
# We do it because node from nodeSet is some testable object, not real
# node, so it has no nodeId and ledger that we can use
upgrader = Upgrader(nodeId=None,
nodeName=None,
dataDir=node.dataLocation,
config=tconf,
ledger=None)
version = bumpedVersion()
ev_data = UpgradeLogData(
datetime.utcnow(), version, 'some_id', tconf.UPGRADE_ENTRY)
upgrader._callUpgradeAgent(ev_data, 1000)
result = loop.run_until_complete(eventuallySoon(_checkFuture(indicator)))
expectedResult = UpgradeMessage(version, tconf.UPGRADE_ENTRY)
assert result == expectedResult.toJson()
def test_upgrade_log_append_api(log_file_path, ev_type):
upgrade_log = UpgradeLog(log_file_path)
ev_data = UpgradeLogData(datetime.datetime.utcnow(), '1.2.3', 'some_id',
'some_pkg')
getattr(upgrade_log, "append_{}".format(ev_type.name))(ev_data)
assert upgrade_log.last_event.data == ev_data
assert upgrade_log.last_event.ev_type == ev_type
def populate_log_with_upgrade_events(pool_txn_node_names,
tdir,
tconf,
version: Tuple[str, str, str],
pkg_name: str = APP_NAME):
for nm in pool_txn_node_names:
config_helper = NodeConfigHelper(nm, tconf, chroot=tdir)
ledger_dir = config_helper.ledger_dir
os.makedirs(ledger_dir)
log = UpgradeLog(os.path.join(ledger_dir, tconf.upgradeLogFile))
when = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc())
ev_data = UpgradeLogData(when, version, randomString(10), pkg_name)
log.append_scheduled(ev_data)
log.append_started(ev_data)
def check_no_loop(nodeSet, ev_type, pkg_name: str = APP_NAME):
for node in nodeSet:
# mimicking upgrade start
node.upgrader._actionLog.append_started(
UpgradeLogData(
datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc()),
node.upgrader.scheduledAction.version,
node.upgrader.scheduledAction.upgrade_id, pkg_name))
node.notify_upgrade_start()
# mimicking upgrader's initialization after restart
node.upgrader.process_action_log_for_first_run()
node.upgrader.scheduledAction = None
assert node.upgrader._actionLog.last_event.ev_type == ev_type
# mimicking node's catchup after restart
node.postConfigLedgerCaughtUp()
assert node.upgrader.scheduledAction is None
assert node.upgrader._actionLog.last_event.ev_type == ev_type
def test_node_sent_upgrade_in_progress(looper, nodeSet, nodeIds, validUpgrade):
'''
Test that each node sends NODE_UPGRADE In Progress event
(because it sees scheduledUpgrade in the Upgrader)
'''
clear_config_ledger(nodeSet)
version = validUpgrade['version']
for node in nodeSet:
node_id = node.poolManager.get_nym_by_name(node.name)
node.upgrader.scheduledAction = UpgradeLogData(
dateutil.parser.parse(validUpgrade['schedule'][node_id]), version,
"upgrade_id", validUpgrade['package'])
node.notify_upgrade_start()
check_node_sent_acknowledges_upgrade(looper,
nodeSet,
nodeIds,
allowed_actions=[IN_PROGRESS],
ledger_size=len(nodeSet),
expected_version=version)
def test_scheduled_once_after_view_change(nodeSet, validUpgrade,
upgradeScheduled):
'''
Test that each node schedules update only once after each view change
'''
# emulate view changes 1-4
emulate_view_change_pool_for_upgrade(nodeSet)
emulate_view_change_pool_for_upgrade(nodeSet)
emulate_view_change_pool_for_upgrade(nodeSet)
emulate_view_change_pool_for_upgrade(nodeSet)
# check that there are no cancel events in Upgrade log
version = validUpgrade['version']
upgrade_id = nodeSet[0].upgrader.scheduledAction.upgrade_id
for node in nodeSet:
node_id = node.poolManager.get_nym_by_name(node.name)
when = dateutil.parser.parse(validUpgrade['schedule'][node_id])
ev_data = UpgradeLogData(when, version, upgrade_id,
validUpgrade['package'])
assert node.upgrader.scheduledAction == ev_data
assert count_action_log_package(list(node.upgrader._actionLog),
validUpgrade['package']) == 1
assert node.upgrader.lastActionEventInfo.ev_type == UpgradeLog.Events.scheduled
assert node.upgrader.lastActionEventInfo.data == ev_data
def testTimeoutWorks(nodeSet, looper, monkeypatch, tconf):
"""
Checks that after some timeout upgrade is marked as failed if
it not started
"""
async def mock(*x):
return None
# patch get_timeout not to wait one whole minute
monkeypatch.setattr(Upgrader, 'get_timeout', lambda self, timeout: timeout)
# patch _open_connection_and_send to make node think it sent upgrade
# successfully
monkeypatch.setattr(Upgrader, '_open_connection_and_send', mock)
pending = {node.name for node in nodeSet}
version = '1.5.1'
timeout = 1
def chk():
nonlocal pending
assert len(pending) == 0
def upgrade_failed_callback_test(nodeName):
nonlocal pending
pending.remove(nodeName)
ev_data = UpgradeLogData(datetime.utcnow(), version, 'some_id', tconf.UPGRADE_ENTRY)
for node in nodeSet:
monkeypatch.setattr(
node.upgrader,
'_actionFailedCallback',
functools.partial(
upgrade_failed_callback_test,
node.name))
looper.run(node.upgrader._sendUpgradeRequest(ev_data, timeout))
looper.run(eventually(chk))
def handleUpgradeTxn(self, txn) -> None:
"""
Handles transaction of type POOL_UPGRADE
Can schedule or cancel upgrade to a newer
version at specified time
:param txn:
"""
FINALIZING_EVENT_TYPES = [
UpgradeLog.Events.succeeded, UpgradeLog.Events.failed
]
if get_type(txn) != POOL_UPGRADE:
return
logger.info("Node '{}' handles upgrade txn {}".format(
self.nodeName, txn))
txn_data = get_payload_data(txn)
action = txn_data[ACTION]
version = txn_data[VERSION]
justification = txn_data.get(JUSTIFICATION)
pkg_name = txn_data.get(PACKAGE, self.config.UPGRADE_ENTRY)
upgrade_id = self.get_action_id(txn)
# TODO test
try:
version = src_version_cls(pkg_name)(version)
except InvalidVersionError as exc:
logger.warning(
"{} can't handle upgrade txn with version {} for package {}: {}"
.format(self, version, pkg_name, exc))
return
if action == START:
# forced txn could have partial schedule list
if self.nodeId not in txn_data[SCHEDULE]:
logger.info("Node '{}' disregards upgrade txn {}".format(
self.nodeName, txn))
return
last_event = self.lastActionEventInfo
if last_event:
if last_event.data.upgrade_id == upgrade_id and last_event.ev_type in FINALIZING_EVENT_TYPES:
logger.info(
"Node '{}' has already performed an upgrade with upgrade_id {}. "
"Last recorded event is {}".format(
self.nodeName, upgrade_id, last_event.data))
return
when = txn_data[SCHEDULE][self.nodeId]
failTimeout = txn_data.get(TIMEOUT, self.defaultActionTimeout)
if isinstance(when, str):
when = dateutil.parser.parse(when)
new_ev_data = UpgradeLogData(when, version, upgrade_id, pkg_name)
if self.scheduledAction:
if self.scheduledAction == new_ev_data:
logger.debug(
"Node {} already scheduled upgrade to version '{}' ".
format(self.nodeName, version))
return
else:
logger.info(
"Node '{}' cancels previous upgrade and schedules a new one to {}"
.format(self.nodeName, version))
self._cancelScheduledUpgrade(justification)
logger.info("Node '{}' schedules upgrade to {}".format(
self.nodeName, version))
self._scheduleUpgrade(new_ev_data, failTimeout)
return
if action == CANCEL:
if self.scheduledAction and self.scheduledAction.version == version:
self._cancelScheduledUpgrade(justification)
logger.info("Node '{}' cancels upgrade to {}".format(
self.nodeName, version))
return
logger.error("Got {} transaction with unsupported action {}".format(
POOL_UPGRADE, action))
def test_upgrade_log_data_pack_unpack():
delimiter = '|'
data = UpgradeLogData(datetime.datetime.utcnow(), '1.2.3', 'some_id',
'some_pkg')
assert data == UpgradeLogData.unpack(data.pack(delimiter=delimiter),
delimiter=delimiter)
def test_upgrade_log_data_unpack_invalid_version():
with pytest.raises(TypeError) as excinfo:
UpgradeLogData(str(datetime.datetime.utcnow()), 123, 'some_id',
'some_pkg')
assert "'version' should be 'SourceVersion' or 'str'" in str(excinfo.value)