def testScheduleNodeUpgrade(tconf, nodeSet): """ Tests that upgrade scheduling works. For that it starts mock control service, schedules upgrade for near future and then checks that service received notification. """ loop = asyncio.get_event_loop() server, indicator = loop.run_until_complete( _createServer( host=tconf.controlServiceHost, port=tconf.controlServicePort ) ) indicator.add_done_callback(_stopServer(server)) node = nodeSet[0] # ATTENTION! nodeId and ledger must not be None, but there # we do not call methods that use them, so we can pass None # We do it because node from nodeSet is some testable object, not real # node, so it has no nodeId and ledger that we can use upgrader = Upgrader(nodeId=None, nodeName=None, dataDir=node.dataLocation, config=tconf, ledger=None) version = bumpedVersion() ev_data = UpgradeLogData( datetime.utcnow(), version, 'some_id', tconf.UPGRADE_ENTRY) upgrader._callUpgradeAgent(ev_data, 1000) result = loop.run_until_complete(eventuallySoon(_checkFuture(indicator))) expectedResult = UpgradeMessage(version, tconf.UPGRADE_ENTRY) assert result == expectedResult.toJson()
def test_upgrade_log_append_api(log_file_path, ev_type): upgrade_log = UpgradeLog(log_file_path) ev_data = UpgradeLogData(datetime.datetime.utcnow(), '1.2.3', 'some_id', 'some_pkg') getattr(upgrade_log, "append_{}".format(ev_type.name))(ev_data) assert upgrade_log.last_event.data == ev_data assert upgrade_log.last_event.ev_type == ev_type
def populate_log_with_upgrade_events(pool_txn_node_names, tdir, tconf, version: Tuple[str, str, str], pkg_name: str = APP_NAME): for nm in pool_txn_node_names: config_helper = NodeConfigHelper(nm, tconf, chroot=tdir) ledger_dir = config_helper.ledger_dir os.makedirs(ledger_dir) log = UpgradeLog(os.path.join(ledger_dir, tconf.upgradeLogFile)) when = datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc()) ev_data = UpgradeLogData(when, version, randomString(10), pkg_name) log.append_scheduled(ev_data) log.append_started(ev_data)
def check_no_loop(nodeSet, ev_type, pkg_name: str = APP_NAME): for node in nodeSet: # mimicking upgrade start node.upgrader._actionLog.append_started( UpgradeLogData( datetime.utcnow().replace(tzinfo=dateutil.tz.tzutc()), node.upgrader.scheduledAction.version, node.upgrader.scheduledAction.upgrade_id, pkg_name)) node.notify_upgrade_start() # mimicking upgrader's initialization after restart node.upgrader.process_action_log_for_first_run() node.upgrader.scheduledAction = None assert node.upgrader._actionLog.last_event.ev_type == ev_type # mimicking node's catchup after restart node.postConfigLedgerCaughtUp() assert node.upgrader.scheduledAction is None assert node.upgrader._actionLog.last_event.ev_type == ev_type
def test_node_sent_upgrade_in_progress(looper, nodeSet, nodeIds, validUpgrade): ''' Test that each node sends NODE_UPGRADE In Progress event (because it sees scheduledUpgrade in the Upgrader) ''' clear_config_ledger(nodeSet) version = validUpgrade['version'] for node in nodeSet: node_id = node.poolManager.get_nym_by_name(node.name) node.upgrader.scheduledAction = UpgradeLogData( dateutil.parser.parse(validUpgrade['schedule'][node_id]), version, "upgrade_id", validUpgrade['package']) node.notify_upgrade_start() check_node_sent_acknowledges_upgrade(looper, nodeSet, nodeIds, allowed_actions=[IN_PROGRESS], ledger_size=len(nodeSet), expected_version=version)
def test_scheduled_once_after_view_change(nodeSet, validUpgrade, upgradeScheduled): ''' Test that each node schedules update only once after each view change ''' # emulate view changes 1-4 emulate_view_change_pool_for_upgrade(nodeSet) emulate_view_change_pool_for_upgrade(nodeSet) emulate_view_change_pool_for_upgrade(nodeSet) emulate_view_change_pool_for_upgrade(nodeSet) # check that there are no cancel events in Upgrade log version = validUpgrade['version'] upgrade_id = nodeSet[0].upgrader.scheduledAction.upgrade_id for node in nodeSet: node_id = node.poolManager.get_nym_by_name(node.name) when = dateutil.parser.parse(validUpgrade['schedule'][node_id]) ev_data = UpgradeLogData(when, version, upgrade_id, validUpgrade['package']) assert node.upgrader.scheduledAction == ev_data assert count_action_log_package(list(node.upgrader._actionLog), validUpgrade['package']) == 1 assert node.upgrader.lastActionEventInfo.ev_type == UpgradeLog.Events.scheduled assert node.upgrader.lastActionEventInfo.data == ev_data
def testTimeoutWorks(nodeSet, looper, monkeypatch, tconf): """ Checks that after some timeout upgrade is marked as failed if it not started """ async def mock(*x): return None # patch get_timeout not to wait one whole minute monkeypatch.setattr(Upgrader, 'get_timeout', lambda self, timeout: timeout) # patch _open_connection_and_send to make node think it sent upgrade # successfully monkeypatch.setattr(Upgrader, '_open_connection_and_send', mock) pending = {node.name for node in nodeSet} version = '1.5.1' timeout = 1 def chk(): nonlocal pending assert len(pending) == 0 def upgrade_failed_callback_test(nodeName): nonlocal pending pending.remove(nodeName) ev_data = UpgradeLogData(datetime.utcnow(), version, 'some_id', tconf.UPGRADE_ENTRY) for node in nodeSet: monkeypatch.setattr( node.upgrader, '_actionFailedCallback', functools.partial( upgrade_failed_callback_test, node.name)) looper.run(node.upgrader._sendUpgradeRequest(ev_data, timeout)) looper.run(eventually(chk))
def handleUpgradeTxn(self, txn) -> None: """ Handles transaction of type POOL_UPGRADE Can schedule or cancel upgrade to a newer version at specified time :param txn: """ FINALIZING_EVENT_TYPES = [ UpgradeLog.Events.succeeded, UpgradeLog.Events.failed ] if get_type(txn) != POOL_UPGRADE: return logger.info("Node '{}' handles upgrade txn {}".format( self.nodeName, txn)) txn_data = get_payload_data(txn) action = txn_data[ACTION] version = txn_data[VERSION] justification = txn_data.get(JUSTIFICATION) pkg_name = txn_data.get(PACKAGE, self.config.UPGRADE_ENTRY) upgrade_id = self.get_action_id(txn) # TODO test try: version = src_version_cls(pkg_name)(version) except InvalidVersionError as exc: logger.warning( "{} can't handle upgrade txn with version {} for package {}: {}" .format(self, version, pkg_name, exc)) return if action == START: # forced txn could have partial schedule list if self.nodeId not in txn_data[SCHEDULE]: logger.info("Node '{}' disregards upgrade txn {}".format( self.nodeName, txn)) return last_event = self.lastActionEventInfo if last_event: if last_event.data.upgrade_id == upgrade_id and last_event.ev_type in FINALIZING_EVENT_TYPES: logger.info( "Node '{}' has already performed an upgrade with upgrade_id {}. " "Last recorded event is {}".format( self.nodeName, upgrade_id, last_event.data)) return when = txn_data[SCHEDULE][self.nodeId] failTimeout = txn_data.get(TIMEOUT, self.defaultActionTimeout) if isinstance(when, str): when = dateutil.parser.parse(when) new_ev_data = UpgradeLogData(when, version, upgrade_id, pkg_name) if self.scheduledAction: if self.scheduledAction == new_ev_data: logger.debug( "Node {} already scheduled upgrade to version '{}' ". format(self.nodeName, version)) return else: logger.info( "Node '{}' cancels previous upgrade and schedules a new one to {}" .format(self.nodeName, version)) self._cancelScheduledUpgrade(justification) logger.info("Node '{}' schedules upgrade to {}".format( self.nodeName, version)) self._scheduleUpgrade(new_ev_data, failTimeout) return if action == CANCEL: if self.scheduledAction and self.scheduledAction.version == version: self._cancelScheduledUpgrade(justification) logger.info("Node '{}' cancels upgrade to {}".format( self.nodeName, version)) return logger.error("Got {} transaction with unsupported action {}".format( POOL_UPGRADE, action))
def test_upgrade_log_data_pack_unpack(): delimiter = '|' data = UpgradeLogData(datetime.datetime.utcnow(), '1.2.3', 'some_id', 'some_pkg') assert data == UpgradeLogData.unpack(data.pack(delimiter=delimiter), delimiter=delimiter)
def test_upgrade_log_data_unpack_invalid_version(): with pytest.raises(TypeError) as excinfo: UpgradeLogData(str(datetime.datetime.utcnow()), 123, 'some_id', 'some_pkg') assert "'version' should be 'SourceVersion' or 'str'" in str(excinfo.value)