Ejemplo n.º 1
0
 def start_proc_monitor(self, name=None, regexp=False, frequency=60):
     if self._procmon is not None:
         self.logger.info('A process monitor is already instantiated')
         return
     self.logger.info('starting process monitoring of ' + name +
                      ' every ' + str(frequency) + 'seconds')
     self._procmon = ProcMonitor(name=name, regexp=regexp,
                                 frequency=frequency)
     self._procmon.start()
Ejemplo n.º 2
0
 def start_proc_monitor(self, name=None, regexp=False, frequency=60):
     if self._procmon is not None:
         self.logger.info("A process monitor is already instantiated")
         return
     self.logger.info("starting process monitoring of " + name + " every " + str(frequency) + "seconds")
     self._procmon = ProcMonitor(name=name, regexp=regexp, frequency=frequency)
     self._procmon.start()
Ejemplo n.º 3
0
    def start_proc_monitor(self, name=None, regexp=False, frequency=60):
        """
        Start the process monitoring

        :param name: Process name
        :type name: str or None
        :param regexp: Regular expression to match
        :type regexp: bool
        :param frequency: Frequency of monitoring
        :type frequency: int
        """
        if self._procmon is not None:
            self.logger.info('A process monitor is already instantiated')
            return
        self.logger.info('starting process monitoring of ' + name +
                         ' every ' + str(frequency) + 'seconds')
        self._procmon = ProcMonitor(name=name, regexp=regexp,
                                    frequency=frequency)
        self._procmon.start()
Ejemplo n.º 4
0
    def start_proc_monitor(self, name=None, regexp=False, frequency=60):
        """
        Start the process monitoring

        :param name: Process name
        :type name: str or None
        :param regexp: Regular expression to match
        :type regexp: bool
        :param frequency: Frequency of monitoring
        :type frequency: int
        """
        if self._procmon is not None:
            self.logger.info('A process monitor is already instantiated')
            return
        self.logger.info('starting process monitoring of ' + name +
                         ' every ' + str(frequency) + 'seconds')
        self._procmon = ProcMonitor(name=name, regexp=regexp,
                                    frequency=frequency)
        self._procmon.start()
Ejemplo n.º 5
0
class PBSTestSuite(unittest.TestCase):

    """
    Generic ``setup``, ``teardown``, and ``logging`` functions to
    be used as parent class for most tests.
    Class instantiates:

    ``server object connected to localhost``

    ``scheduler objected connected to localhost``

    ``mom object connected to localhost``

    Custom parameters:

    :param server: The hostname on which the PBS ``server/scheduler``
                   are running
    :param mom: The hostname on which the PBS MoM is running
    :param servers: Colon-separated list of hostnames hosting a PBS server.
                    Servers are then accessible as a dictionary in the
                    instance variable servers.
    :param client: For CLI mode only, name of the host on which the PBS
                   client commands are to be run from. Format is
                   ``<host>@<path-to-config-file>``
    :param moms: Colon-separated list of hostnames hosting a PBS MoM.
                 MoMs are made accessible as a dictionary in the instance
                 variable moms.
    :param comms: Colon-separated list of hostnames hosting a PBS Comm.
                  Comms are made accessible as a dictionary in the
                  instance variable comms.
    :param nomom=<host1>\:<host2>...: expect no MoM on given set of hosts
    :param mode: Sets mode of operation to PBS server. Can be either
                 ``'cli'`` or ``'api'``.Defaults to API behavior.
    :param conn_timeout: set a timeout in seconds after which a pbs_connect
                         IFL call is refreshed (i.e., disconnected)
    :param skip-setup: Bypasses setUp of PBSTestSuite (not custom ones)
    :param skip-teardown: Bypasses tearDown of PBSTestSuite (not custom ones)
    :param procinfo: Enables process monitoring thread, logged into
                     ptl_proc_info test metrics. The value can be set to
                     _all_ to monitor all PBS processes,including
                     ``pbs_server``, ``pbs_sched``, ``pbs_mom``, or a process
                     defined by name.
    :param revert-to-defaults=<True|False>: if False, will not revert to
                                            defaults.True by default.
    :param revert-hooks=<True|False>: if False, do not revert hooks to
                                      defaults.Defaults to True.
                                      ``revert-to-defaults`` set to False
                                      overrides this setting.
    :param del-hooks=<True|False>: If False, do not delete hooks. Defaults
                                   to False.``revert-to-defaults`` set to
                                   False overrides this setting.
    :param revert-queues=<True|False>: If False, do not revert queues to
                                       defaults.Defaults to True.
                                       ``revert-to-defaults`` set to False
                                       overrides this setting.
    :param revert-resources=<True|False>: If False, do not revert resources
                                          to defaults. Defaults to True.
                                          ``revert-to-defaults`` set to False
                                          overrides this setting.
    :param del-queues=<True|False>: If False, do not delete queues. Defaults
                                    to False.``revert-to-defaults`` set to
                                    Falseoverrides this setting.
    :param del-vnodes=<True|False>: If False, do not delete vnodes on MoM
                                    instances.Defaults to True.
    :param server-revert-to-defaults=<True|False>: if False, don't revert
                                                   Server to defaults
    :param comm-revert-to-defaults=<True|False>: if False, don't revert Comm
                                                 to defaults
    :param mom-revert-to-defaults=<True|False>: if False, don't revert MoM
                                                to defaults
    :param sched-revert-to-defaults=<True|False>: if False, don't revert
                                                  Scheduler to defaults
    :param procmon: Enables process monitoring. Multiple values must be
                    colon separated. For example to monitor ``server``,
                    ``sched``, and ``mom`` use
                    ``procmon=pbs_server:pbs_sched:pbs_mom``
    :param procmon-freq: Sets a polling frequency for the process monitoring
                         tool.Defaults to 10 seconds.
    :param test-users: colon-separated list of users to use as test users.
                       The users specified override the default users in the
                       order in which they appear in the ``PBS_USERS`` list.
    :param default-testcase-timeout: Default test case timeout value.
    :param data-users: colon-separated list of data users.
    :param oper-users: colon-separated list of operator users.
    :param mgr-users: colon-separated list of manager users.
    :param root-users: colon-separated list of root users.
    :param build-users: colon-separated list of build users.
    :param clienthost: the hostnames to set in the MoM config file
    """

    logger = logging.getLogger(__name__)
    metrics_data = {}
    conf = {}
    param = None
    du = DshUtils()
    _procmon = None
    _process_monitoring = False
    revert_to_defaults = True
    server_revert_to_defaults = True
    mom_revert_to_defaults = True
    sched_revert_to_defaults = True
    revert_queues = True
    revert_resources = True
    revert_hooks = True
    del_hooks = True
    del_queues = True
    del_vnodes = True
    server = None
    scheduler = None
    mom = None
    comm = None
    servers = None
    schedulers = None
    moms = None
    comms = None

    @classmethod
    def setUpClass(cls):
        cls.log_enter_setup(True)
        cls._testMethodName = 'setUpClass'
        cls.parse_param()
        cls.init_param()
        cls.check_users_exist()
        cls.init_servers()
        cls.init_comms()
        cls.init_schedulers()
        cls.init_moms()
        cls.log_end_setup(True)

    def setUp(self):
        if 'skip-setup' in self.conf:
            return
        self.log_enter_setup()
        self.init_proc_mon()
        self.revert_servers()
        self.revert_comms()
        self.revert_schedulers()
        self.revert_moms()
        self.log_end_setup()

    @classmethod
    def log_enter_setup(cls, iscls=False):
        _m = ' Entered ' + cls.__name__ + ' setUp'
        if iscls:
            _m += 'Class'
        _m_len = len(_m)
        cls.logger.info('=' * _m_len)
        cls.logger.info(_m)
        cls.logger.info('=' * _m_len)

    @classmethod
    def log_end_setup(cls, iscls=False):
        _m = 'Completed ' + cls.__name__ + ' setUp'
        if iscls:
            _m += 'Class'
        _m_len = len(_m)
        cls.logger.info('=' * _m_len)
        cls.logger.info(_m)
        cls.logger.info('=' * _m_len)

    @classmethod
    def _validate_param(cls, pname):
        """
        Check if parameter was enabled at the ``command-line``

        :param pname: parameter name
        :type pname: str
        :param pvar: class variable to set according to command-line setting
        """
        if pname not in cls.conf:
            return
        if cls.conf[pname] in PTL_TRUE:
            setattr(cls, pname.replace('-', '_'), True)
        else:
            setattr(cls, pname.replace('-', '_'), False)

    @classmethod
    def _set_user(cls, name, user_list):
        if name in cls.conf:
            for idx, u in enumerate(cls.conf[name].split(':')):
                user_list[idx].__init__(u)

    @classmethod
    def check_users_exist(cls):
        """
        Check whether the user is exist or not
        """
        testusersexist = True
        for u in [TEST_USER, TEST_USER1, TEST_USER2, TEST_USER3]:
            rv = cls.du.check_user_exists(str(u))
            if not rv:
                _msg = 'User ' + str(u) + ' does not exist!'
                raise setUpClassError(_msg)
        return testusersexist

    @classmethod
    def kicksched_action(cls, server, obj_type, *args, **kwargs):
        """
        custom scheduler action to kick a scheduling cycle when expectig
        a job state change
        """
        if server is None:
            cls.logger.error('no server defined for custom action')
            return
        if obj_type == JOB:
            if (('scheduling' in server.attributes) and
                    (server.attributes['scheduling'] != 'False')):
                server.manager(MGR_CMD_SET, MGR_OBJ_SERVER,
                               {'scheduling': 'True'},
                               level=logging.DEBUG)

    @classmethod
    def parse_param(cls):
        """
        get test configuration parameters as a ``comma-separated``
        list of attributes.

        Attributes may be ``'='`` separated key value pairs or standalone
        entries.

        ``Multi-property`` attributes are colon-delimited.
        """
        if cls.param is None:
            return
        for h in cls.param.split(','):
            if '=' in h:
                k, v = h.split('=')
                cls.conf[k.strip()] = v.strip()
            else:
                cls.conf[h.strip()] = ''
        if (('clienthost' in cls.conf) and
                not isinstance(cls.conf['clienthost'], list)):
            cls.conf['clienthost'] = cls.conf['clienthost'].split(':')
        users_map = [('test-users', PBS_USERS),
                     ('oper-users', PBS_OPER_USERS),
                     ('mgr-users', PBS_MGR_USERS),
                     ('data-users', PBS_DATA_USERS),
                     ('root-users', PBS_ROOT_USERS),
                     ('build-users', PBS_BUILD_USERS)]
        for k, v in users_map:
            cls._set_user(k, v)

    @classmethod
    def init_param(cls):
        cls._validate_param('revert-to-defaults')
        cls._validate_param('server-revert-to-defaults')
        cls._validate_param('comm-revert-to-defaults')
        cls._validate_param('mom-revert-to-defaults')
        cls._validate_param('sched-revert-to-defaults')
        cls._validate_param('del-hooks')
        cls._validate_param('revert-hooks')
        cls._validate_param('del-queues')
        cls._validate_param('del-vnodes')
        cls._validate_param('revert-queues')
        cls._validate_param('revert-resources')
        if 'default-testcase-timeout' not in cls.conf.keys():
            cls.conf['default_testcase_timeout'] = 180
        else:
            cls.conf['default_testcase_timeout'] = int(
                cls.conf['default-testcase-timeout'])

    @classmethod
    def is_server_licensed(cls, server):
        """
        Check if server is licensed or not
        """
        for i in range(0, 10, 1):
            lic = server.status(SERVER, 'license_count', level=logging.INFOCLI)
            if lic and 'license_count' in lic[0]:
                lic = PbsTypeLicenseCount(lic[0]['license_count'])
                if int(lic['Avail_Sockets']) > 0:
                    return True
                elif int(lic['Avail_Global']) > 0:
                    return True
                elif int(lic['Avail_Local']) > 0:
                    return True
            time.sleep(i)
        return False

    @classmethod
    def init_from_conf(cls, conf, single=None, multiple=None, skip=None,
                       func=None):
        """
        Helper method to parse test parameters for`` mom/server/scheduler``
        instances.

        The supported format of each service request is:

        ``hostname@configuration/path``

        For example:

        ``pbs_benchpress -p server=remote@/etc/pbs.conf.12.0``

        initializes a remote server instance that is configured according to
        the remote file ``/etc/pbs.conf.12.0``
        """
        endpoints = []
        if ((multiple in conf) and (conf[multiple] is not None)):
            __objs = conf[multiple].split(':')
            for _m in __objs:
                tmp = _m.split('@')
                if len(tmp) == 2:
                    endpoints.append(tuple(tmp))
                elif len(tmp) == 1:
                    endpoints.append((tmp[0], None))
        elif ((single in conf) and (conf[single] is not None)):
            tmp = conf[single].split('@')
            if len(tmp) == 2:
                endpoints.append(tuple(tmp))
            elif len(tmp) == 1:
                endpoints.append((tmp[0], None))
        else:
            endpoints = [(socket.gethostname(), None)]
        objs = PBSServiceInstanceWrapper()
        for name, objconf in endpoints:
            if ((skip is not None) and (skip in conf) and
                    ((name in conf[skip]) or (conf[skip] in name))):
                continue
            if objconf is not None:
                n = name + '@' + objconf
            else:
                n = name
            if getattr(cls, "server", None) is not None:
                objs[n] = func(name, pbsconf_file=objconf,
                               server=cls.server.hostname)
            else:
                objs[n] = func(name, pbsconf_file=objconf)
            if objs[n] is None:
                _msg = 'Failed %s(%s, %s)' % (func.__name__, name, objconf)
                raise setUpClassError(_msg)
        return objs

    @classmethod
    def init_servers(cls, init_server_func=None, skip=None):
        """
        Initialize servers
        """
        if init_server_func is None:
            init_server_func = cls.init_server
        if 'servers' in cls.conf:
            if 'comms' not in cls.conf:
                cls.conf['comms'] = cls.conf['servers']
            if 'schedulers' not in cls.conf:
                cls.conf['schedulers'] = cls.conf['servers']
            if 'moms' not in cls.conf:
                cls.conf['moms'] = cls.conf['servers']
        if 'server' in cls.conf:
            if 'comm' not in cls.conf:
                cls.conf['comm'] = cls.conf['server']
            if 'scheduler' not in cls.conf:
                cls.conf['scheduler'] = cls.conf['server']
            if 'mom' not in cls.conf:
                cls.conf['mom'] = cls.conf['server']
        cls.servers = cls.init_from_conf(conf=cls.conf, single='server',
                                         multiple='servers', skip=skip,
                                         func=init_server_func)
        if cls.servers:
            cls.server = cls.servers.values()[0]

    @classmethod
    def init_comms(cls, init_comm_func=None, skip=None):
        """
        Initialize comms
        """
        if init_comm_func is None:
            init_comm_func = cls.init_comm
        cls.comms = cls.init_from_conf(conf=cls.conf,
                                       single='comm',
                                       multiple='comms', skip=skip,
                                       func=init_comm_func)
        if cls.comms:
            cls.comm = cls.comms.values()[0]

    @classmethod
    def init_schedulers(cls, init_sched_func=None, skip=None):
        """
        Initialize schedulers
        """
        if init_sched_func is None:
            init_sched_func = cls.init_scheduler
        cls.schedulers = cls.init_from_conf(conf=cls.conf,
                                            single='scheduler',
                                            multiple='schedulers', skip=skip,
                                            func=init_sched_func)
        if cls.schedulers:
            cls.scheduler = cls.schedulers.values()[0]

    @classmethod
    def init_moms(cls, init_mom_func=None, skip='nomom'):
        """
        Initialize moms
        """
        if init_mom_func is None:
            init_mom_func = cls.init_mom
        cls.moms = cls.init_from_conf(conf=cls.conf, single='mom',
                                      multiple='moms', skip=skip,
                                      func=init_mom_func)
        if cls.moms:
            cls.mom = cls.moms.values()[0]

    @classmethod
    def init_server(cls, hostname, pbsconf_file=None):
        """
        Initialize a server instance

        Define custom expect action to trigger a scheduling cycle when job
        is not in running state

        :returns: The server instance on success and None on failure
        """
        client = hostname
        client_conf = None
        if 'client' in cls.conf:
            _cl = cls.conf['client'].split('@')
            client = _cl[0]
            if len(_cl) > 1:
                client_conf = _cl[1]
        server = Server(hostname, pbsconf_file=pbsconf_file, client=client,
                        client_pbsconf_file=client_conf)
        server._conn_timeout = 0
        if cls.conf is not None:
            if 'mode' in cls.conf:
                if cls.conf['mode'] == 'cli':
                    server.set_op_mode(PTL_CLI)
            if 'conn_timeout' in cls.conf:
                conn_timeout = int(cls.conf['conn_timeout'])
                server.set_connect_timeout(conn_timeout)
        sched_action = ExpectAction('kicksched', True, JOB,
                                    cls.kicksched_action)
        server.add_expect_action(action=sched_action)
        return server

    @classmethod
    def init_comm(cls, hostname, pbsconf_file=None, server=None):
        """
        Initialize a Comm instance associated to the given hostname.

        This method must be called after init_server

        :param hostname: The host on which the Comm is running
        :type hostname: str
        :param pbsconf_file: Optional path to an alternate pbs config file
        :type pbsconf_file: str or None
        :returns: The instantiated Comm upon success and None on failure.
        :param server: The server name associated to the Comm
        :type server: str
        Return the instantiated Comm upon success and None on failure.
        """
        try:
            server = cls.servers[server]
        except:
            server = None
        return Comm(hostname, pbsconf_file=pbsconf_file, server=server)

    @classmethod
    def init_scheduler(cls, hostname, pbsconf_file=None, server=None):
        """
        Initialize a Scheduler instance associated to the given server.
        This method must be called after ``init_server``

        :param server: The server name associated to the scheduler
        :type server: str
        :param pbsconf_file: Optional path to an alternate config file
        :type pbsconf_file: str or None
        :param hostname: The host on which Sched is running
        :type hostname: str
        :returns: The instantiated scheduler upon success and None on failure
        """
        try:
            server = cls.servers[server]
        except:
            server = None
        return Scheduler(hostname=hostname, server=server,
                         pbsconf_file=pbsconf_file)

    @classmethod
    def init_mom(cls, hostname, pbsconf_file=None, server=None):
        """
        Initialize a ``MoM`` instance associated to the given hostname.

        This method must be called after ``init_server``

        :param hostname: The host on which the MoM is running
        :type hostname: str
        :param pbsconf_file: Optional path to an alternate pbs config file
        :type pbsconf_file: str or None
        :returns: The instantiated MoM upon success and None on failure.
        """
        try:
            server = cls.servers[server]
        except:
            server = None
        return MoM(hostname, pbsconf_file=pbsconf_file, server=server)

    def init_proc_mon(self):
        """
        Initialize process monitoring when requested
        """
        if 'procmon' in self.conf:
            _proc_mon = []
            for p in self.conf['procmon'].split(':'):
                _proc_mon += ['.*' + p + '.*']
            if _proc_mon:
                if 'procmon-freq' in self.conf:
                    freq = int(self.conf['procmon-freq'])
                else:
                    freq = 10
                self.start_proc_monitor(name='|'.join(_proc_mon), regexp=True,
                                        frequency=freq)
                self._process_monitoring = True

    def revert_servers(self, force=False):
        """
        Revert the values set for servers
        """
        for server in self.servers.values():
            self.revert_server(server, force)

    def revert_comms(self, force=False):
        """
        Revert the values set for comms
        """
        for comm in self.comms.values():
            self.revert_comm(comm, force)

    def revert_schedulers(self, force=False):
        """
        Revert the values set for schedulers
        """
        for sched in self.schedulers.values():
            self.revert_scheduler(sched, force)

    def revert_moms(self, force=False):
        """
        Revert the values set for moms
        """
        for mom in self.moms.values():
            self.revert_mom(mom, force)

    def revert_server(self, server, force=False):
        """
        Revert the values set for server
        """
        rv = server.isUp()
        if not rv:
            self.logger.error('server ' + server.hostname + ' is down')
            server.start()
            msg = 'Failed to restart server ' + server.hostname
            self.assertTrue(server.isUp(), msg)
        server_stat = server.status(SERVER)[0]
        current_user = pwd.getpwuid(os.getuid())[0]
        try:
            # remove current user's entry from managers list (if any)
            a = {ATTR_managers: (DECR, current_user + '@*')}
            server.manager(MGR_CMD_SET, SERVER, a, sudo=True)
        except:
            pass
        a = {ATTR_managers: (INCR, current_user + '@*')}
        server.manager(MGR_CMD_SET, SERVER, a, sudo=True)
        if ((self.revert_to_defaults and self.server_revert_to_defaults) or
                force):
            server.revert_to_defaults(reverthooks=self.revert_hooks,
                                      delhooks=self.del_hooks,
                                      revertqueues=self.revert_queues,
                                      delqueues=self.del_queues,
                                      revertresources=self.revert_resources,
                                      server_stat=server_stat)
        rv = self.is_server_licensed(server)
        _msg = 'No license found on server %s' % (server.shortname)
        self.assertTrue(rv, _msg)
        self.logger.info('server: %s licensed', server.hostname)

    def revert_comm(self, comm, force=False):
        """
        Revert the values set for comm
        """
        rv = comm.isUp()
        if not rv:
            self.logger.error('comm ' + comm.hostname + ' is down')
            comm.start()
            msg = 'Failed to restart comm ' + comm.hostname
            self.assertTrue(comm.isUp(), msg)

    def revert_scheduler(self, scheduler, force=False):
        """
        Revert the values set for scheduler
        """
        rv = scheduler.isUp()
        if not rv:
            self.logger.error('scheduler ' + scheduler.hostname + ' is down')
            scheduler.start()
            msg = 'Failed to restart scheduler ' + scheduler.hostname
            self.assertTrue(scheduler.isUp(), msg)
        if ((self.revert_to_defaults and self.sched_revert_to_defaults) or
                force):
            rv = scheduler.revert_to_defaults()
            _msg = 'Failed to revert sched %s' % (scheduler.hostname)
            self.assertTrue(rv, _msg)

    def revert_mom(self, mom, force=False):
        """
        Revert the values set for mom

        :param mom: the MoM object whose values are to be reverted
        :type mom: MoM object
        :param force: Option to reverse forcibly
        :type force: bool
        """
        rv = mom.isUp()
        if not rv:
            self.logger.error('mom ' + mom.hostname + ' is down')
            mom.start()
            msg = 'Failed to restart mom ' + mom.hostname
            self.assertTrue(mom.isUp(), msg)
        mom.pbs_version()
        if ((self.revert_to_defaults and self.mom_revert_to_defaults) or
                force):
            rv = mom.revert_to_defaults(delvnodedefs=self.del_vnodes)
            _msg = 'Failed to revert mom %s' % (mom.hostname)
            self.assertTrue(rv, _msg)
            if 'clienthost' in self.conf:
                mom.add_config({'$clienthost': self.conf['clienthost']})
            a = {'state': 'free', 'resources_available.ncpus': (GE, 1)}
            nodes = self.server.counter(NODE, a, attrop=PTL_AND,
                                        level=logging.DEBUG)
            if not nodes:
                try:
                    self.server.manager(MGR_CMD_DELETE, NODE, None, '')
                except:
                    pass
                mom.delete_vnode_defs()
                mom.delete_vnodes()
                mom.restart()
                self.logger.info('server: no nodes defined, creating one')
                self.server.manager(MGR_CMD_CREATE, NODE, None, mom.shortname)
        name = mom.shortname
        try:
            self.server.status(NODE, id=name)
        except PbsStatusError:
            # server doesn't have node with shortname
            # check with hostname
            name = mom.hostname
            try:
                self.server.status(NODE, id=name)
            except PbsStatusError:
                # server doesn't have node for this mom yet
                # so create with shortname
                name = mom.shortname
                self.server.manager(MGR_CMD_CREATE, NODE, None, mom.shortname)
        self.server.expect(NODE, {ATTR_NODE_state: 'free'}, id=name,
                           interval=1)
        return mom

    def analyze_logs(self):
        """
        analyze accounting and scheduler logs from time test was started
        until it finished
        """
        pla = PBSLogAnalyzer()
        self.metrics_data = pla.analyze_logs(serverlog=self.server.logfile,
                                             schedlog=self.scheduler.logfile,
                                             momlog=self.mom.logfile,
                                             acctlog=self.server.acctlogfile,
                                             start=self.server.ctime,
                                             end=int(time.time()))

    def start_proc_monitor(self, name=None, regexp=False, frequency=60):
        """
        Start the process monitoring

        :param name: Process name
        :type name: str or None
        :param regexp: Regular expression to match
        :type regexp: bool
        :param frequency: Frequency of monitoring
        :type frequency: int
        """
        if self._procmon is not None:
            self.logger.info('A process monitor is already instantiated')
            return
        self.logger.info('starting process monitoring of ' + name +
                         ' every ' + str(frequency) + 'seconds')
        self._procmon = ProcMonitor(name=name, regexp=regexp,
                                    frequency=frequency)
        self._procmon.start()

    def stop_proc_monitor(self):
        """
        Stop the process monitoring
        """
        if not self._process_monitoring:
            return
        self.logger.info('stopping process monitoring')
        self._procmon.stop()
        self.metrics_data['procs'] = self._procmon.db_proc_info
        self._process_monitoring = False

    def skipTest(self, reason=None):
        """
        Skip Test

        :param reason: message to indicate why test is skipped
        :type reason: str or None
        """
        if reason:
            self.logger.warning('test skipped: ' + reason)
        else:
            reason = 'unknown'
        raise SkipTest(reason)

    skip_test = skipTest

    @classmethod
    def log_enter_teardown(cls, iscls=False):
        _m = ' Entered ' + cls.__name__ + ' tearDown'
        if iscls:
            _m += 'Class'
        _m_len = len(_m)
        cls.logger.info('=' * _m_len)
        cls.logger.info(_m)
        cls.logger.info('=' * _m_len)

    @classmethod
    def log_end_teardown(cls, iscls=False):
        _m = 'Completed ' + cls.__name__ + ' tearDown'
        if iscls:
            _m += 'Class'
        _m_len = len(_m)
        cls.logger.info('=' * _m_len)
        cls.logger.info(_m)
        cls.logger.info('=' * _m_len)

    def tearDown(self):
        """
        verify that ``server`` and ``scheduler`` are up
        clean up jobs and reservations
        """
        if 'skip-teardown' in self.conf:
            return
        self.log_enter_teardown()
        self.stop_proc_monitor()
        self.log_end_teardown()

    @classmethod
    def tearDownClass(cls):
        cls._testMethodName = 'tearDownClass'
Ejemplo n.º 6
0
class PBSTestSuite(unittest.TestCase):
    """
    Generic ``setup``, ``teardown``, and ``logging`` functions to
    be used as parent class for most tests.
    Class instantiates:

    ``server object connected to localhost``

    ``scheduler objected connected to localhost``

    ``mom object connected to localhost``

    Custom parameters:

    :param server: The hostname on which the PBS ``server/scheduler``
                   are running
    :param mom: The hostname on which the PBS MoM is running
    :param servers: Colon-separated list of hostnames hosting a PBS server.
                    Servers are then accessible as a dictionary in the
                    instance variable servers.
    :param client: For CLI mode only, name of the host on which the PBS
                   client commands are to be run from. Format is
                   ``<host>@<path-to-config-file>``
    :param moms: Colon-separated list of hostnames hosting a PBS MoM.
                 MoMs are made accessible as a dictionary in the instance
                 variable moms.
    :param comms: Colon-separated list of hostnames hosting a PBS Comm.
                  Comms are made accessible as a dictionary in the
                  instance variable comms.
    :param nomom=<host1>\:<host2>...: expect no MoM on given set of hosts
    :param mode: Sets mode of operation to PBS server. Can be either
                 ``'cli'`` or ``'api'``.Defaults to API behavior.
    :param conn_timeout: set a timeout in seconds after which a pbs_connect
                         IFL call is refreshed (i.e., disconnected)
    :param skip-setup: Bypasses setUp of PBSTestSuite (not custom ones)
    :param skip-teardown: Bypasses tearDown of PBSTestSuite (not custom ones)
    :param procinfo: Enables process monitoring thread, logged into
                     ptl_proc_info test metrics. The value can be set to
                     _all_ to monitor all PBS processes,including
                     ``pbs_server``, ``pbs_sched``, ``pbs_mom``, or a process
                     defined by name.
    :param revert-to-defaults=<True|False>: if False, will not revert to
                                            defaults.True by default.
    :param revert-hooks=<True|False>: if False, do not revert hooks to
                                      defaults.Defaults to True.
                                      ``revert-to-defaults`` set to False
                                      overrides this setting.
    :param del-hooks=<True|False>: If False, do not delete hooks. Defaults
                                   to False.``revert-to-defaults`` set to
                                   False overrides this setting.
    :param revert-queues=<True|False>: If False, do not revert queues to
                                       defaults.Defaults to True.
                                       ``revert-to-defaults`` set to False
                                       overrides this setting.
    :param revert-resources=<True|False>: If False, do not revert resources
                                          to defaults. Defaults to True.
                                          ``revert-to-defaults`` set to False
                                          overrides this setting.
    :param del-queues=<True|False>: If False, do not delete queues. Defaults
                                    to False.``revert-to-defaults`` set to
                                    Falseoverrides this setting.
    :param del-vnodes=<True|False>: If False, do not delete vnodes on MoM
                                    instances.Defaults to True.
    :param server-revert-to-defaults=<True|False>: if False, don't revert
                                                   Server to defaults
    :param comm-revert-to-defaults=<True|False>: if False, don't revert Comm
                                                 to defaults
    :param mom-revert-to-defaults=<True|False>: if False, don't revert MoM
                                                to defaults
    :param sched-revert-to-defaults=<True|False>: if False, don't revert
                                                  Scheduler to defaults
    :param procmon: Enables process monitoring. Multiple values must be
                    colon separated. For example to monitor ``server``,
                    ``sched``, and ``mom`` use
                    ``procmon=pbs_server:pbs_sched:pbs_mom``
    :param procmon-freq: Sets a polling frequency for the process monitoring
                         tool.Defaults to 10 seconds.
    :param test-users: colon-separated list of users to use as test users.
                       The users specified override the default users in the
                       order in which they appear in the ``PBS_USERS`` list.
    :param default-testcase-timeout: Default test case timeout value.
    :param data-users: colon-separated list of data users.
    :param oper-users: colon-separated list of operator users.
    :param mgr-users: colon-separated list of manager users.
    :param root-users: colon-separated list of root users.
    :param build-users: colon-separated list of build users.
    :param clienthost: the hostnames to set in the MoM config file
    """

    logger = logging.getLogger(__name__)
    metrics_data = {}
    conf = {}
    param = None
    du = DshUtils()
    _procmon = None
    _process_monitoring = False
    revert_to_defaults = True
    server_revert_to_defaults = True
    mom_revert_to_defaults = True
    sched_revert_to_defaults = True
    revert_queues = True
    revert_resources = True
    revert_hooks = True
    del_hooks = True
    del_queues = True
    del_vnodes = True
    server = None
    scheduler = None
    mom = None
    comm = None
    servers = None
    schedulers = None
    moms = None
    comms = None

    @classmethod
    def setUpClass(cls):
        cls.log_enter_setup(True)
        cls._testMethodName = 'setUpClass'
        cls.parse_param()
        cls.init_param()
        cls.check_users_exist()
        cls.init_servers()
        cls.init_comms()
        cls.init_schedulers()
        cls.init_moms()
        cls.log_end_setup(True)

    def setUp(self):
        if 'skip-setup' in self.conf:
            return
        self.log_enter_setup()
        self.init_proc_mon()
        self.revert_servers()
        self.revert_comms()
        self.revert_schedulers()
        self.revert_moms()
        self.log_end_setup()

    @classmethod
    def log_enter_setup(cls, iscls=False):
        _m = ' Entered ' + cls.__name__ + ' setUp'
        if iscls:
            _m += 'Class'
        _m_len = len(_m)
        cls.logger.info('=' * _m_len)
        cls.logger.info(_m)
        cls.logger.info('=' * _m_len)

    @classmethod
    def log_end_setup(cls, iscls=False):
        _m = 'Completed ' + cls.__name__ + ' setUp'
        if iscls:
            _m += 'Class'
        _m_len = len(_m)
        cls.logger.info('=' * _m_len)
        cls.logger.info(_m)
        cls.logger.info('=' * _m_len)

    @classmethod
    def _validate_param(cls, pname):
        """
        Check if parameter was enabled at the ``command-line``

        :param pname: parameter name
        :type pname: str
        :param pvar: class variable to set according to command-line setting
        """
        if pname not in cls.conf:
            return
        if cls.conf[pname] in PTL_TRUE:
            setattr(cls, pname.replace('-', '_'), True)
        else:
            setattr(cls, pname.replace('-', '_'), False)

    @classmethod
    def _set_user(cls, name, user_list):
        if name in cls.conf:
            for idx, u in enumerate(cls.conf[name].split(':')):
                user_list[idx].__init__(u)

    @classmethod
    def check_users_exist(cls):
        """
        Check whether the user is exist or not
        """
        testusersexist = True
        for u in [TEST_USER, TEST_USER1, TEST_USER2, TEST_USER3]:
            rv = cls.du.check_user_exists(str(u))
            if not rv:
                _msg = 'User ' + str(u) + ' does not exist!'
                raise setUpClassError(_msg)
        return testusersexist

    @classmethod
    def kicksched_action(cls, server, obj_type, *args, **kwargs):
        """
        custom scheduler action to kick a scheduling cycle when expectig
        a job state change
        """
        if server is None:
            cls.logger.error('no server defined for custom action')
            return
        if obj_type == JOB:
            if (('scheduling' in server.attributes)
                    and (server.attributes['scheduling'] != 'False')):
                server.manager(MGR_CMD_SET,
                               MGR_OBJ_SERVER, {'scheduling': 'True'},
                               level=logging.DEBUG)

    @classmethod
    def parse_param(cls):
        """
        get test configuration parameters as a ``comma-separated``
        list of attributes.

        Attributes may be ``'='`` separated key value pairs or standalone
        entries.

        ``Multi-property`` attributes are colon-delimited.
        """
        if cls.param is None:
            return
        for h in cls.param.split(','):
            if '=' in h:
                k, v = h.split('=')
                cls.conf[k.strip()] = v.strip()
            else:
                cls.conf[h.strip()] = ''
        if (('clienthost' in cls.conf)
                and not isinstance(cls.conf['clienthost'], list)):
            cls.conf['clienthost'] = cls.conf['clienthost'].split(':')
        users_map = [('test-users', PBS_USERS), ('oper-users', PBS_OPER_USERS),
                     ('mgr-users', PBS_MGR_USERS),
                     ('data-users', PBS_DATA_USERS),
                     ('root-users', PBS_ROOT_USERS),
                     ('build-users', PBS_BUILD_USERS)]
        for k, v in users_map:
            cls._set_user(k, v)

    @classmethod
    def init_param(cls):
        cls._validate_param('revert-to-defaults')
        cls._validate_param('server-revert-to-defaults')
        cls._validate_param('comm-revert-to-defaults')
        cls._validate_param('mom-revert-to-defaults')
        cls._validate_param('sched-revert-to-defaults')
        cls._validate_param('del-hooks')
        cls._validate_param('revert-hooks')
        cls._validate_param('del-queues')
        cls._validate_param('del-vnodes')
        cls._validate_param('revert-queues')
        cls._validate_param('revert-resources')
        if 'default-testcase-timeout' not in cls.conf.keys():
            cls.conf['default_testcase_timeout'] = 180
        else:
            cls.conf['default_testcase_timeout'] = int(
                cls.conf['default-testcase-timeout'])

    @classmethod
    def is_server_licensed(cls, server):
        """
        Check if server is licensed or not
        """
        for i in range(0, 10, 1):
            lic = server.status(SERVER, 'license_count', level=logging.INFOCLI)
            if lic and 'license_count' in lic[0]:
                lic = PbsTypeLicenseCount(lic[0]['license_count'])
                if int(lic['Avail_Sockets']) > 0:
                    return True
                elif int(lic['Avail_Global']) > 0:
                    return True
                elif int(lic['Avail_Local']) > 0:
                    return True
            time.sleep(i)
        return False

    @classmethod
    def init_from_conf(cls,
                       conf,
                       single=None,
                       multiple=None,
                       skip=None,
                       func=None):
        """
        Helper method to parse test parameters for`` mom/server/scheduler``
        instances.

        The supported format of each service request is:

        ``hostname@configuration/path``

        For example:

        ``pbs_benchpress -p server=remote@/etc/pbs.conf.12.0``

        initializes a remote server instance that is configured according to
        the remote file ``/etc/pbs.conf.12.0``
        """
        endpoints = []
        if ((multiple in conf) and (conf[multiple] is not None)):
            __objs = conf[multiple].split(':')
            for _m in __objs:
                tmp = _m.split('@')
                if len(tmp) == 2:
                    endpoints.append(tuple(tmp))
                elif len(tmp) == 1:
                    endpoints.append((tmp[0], None))
        elif ((single in conf) and (conf[single] is not None)):
            tmp = conf[single].split('@')
            if len(tmp) == 2:
                endpoints.append(tuple(tmp))
            elif len(tmp) == 1:
                endpoints.append((tmp[0], None))
        else:
            endpoints = [(socket.gethostname(), None)]
        objs = PBSServiceInstanceWrapper()
        for name, objconf in endpoints:
            if ((skip is not None) and (skip in conf)
                    and ((name in conf[skip]) or (conf[skip] in name))):
                continue
            if objconf is not None:
                n = name + '@' + objconf
            else:
                n = name
            if getattr(cls, "server", None) is not None:
                objs[n] = func(name,
                               pbsconf_file=objconf,
                               server=cls.server.hostname)
            else:
                objs[n] = func(name, pbsconf_file=objconf)
            if objs[n] is None:
                _msg = 'Failed %s(%s, %s)' % (func.__name__, name, objconf)
                raise setUpClassError(_msg)
        return objs

    @classmethod
    def init_servers(cls, init_server_func=None, skip=None):
        """
        Initialize servers
        """
        if init_server_func is None:
            init_server_func = cls.init_server
        if 'servers' in cls.conf:
            if 'comms' not in cls.conf:
                cls.conf['comms'] = cls.conf['servers']
            if 'schedulers' not in cls.conf:
                cls.conf['schedulers'] = cls.conf['servers']
            if 'moms' not in cls.conf:
                cls.conf['moms'] = cls.conf['servers']
        if 'server' in cls.conf:
            if 'comm' not in cls.conf:
                cls.conf['comm'] = cls.conf['server']
            if 'scheduler' not in cls.conf:
                cls.conf['scheduler'] = cls.conf['server']
            if 'mom' not in cls.conf:
                cls.conf['mom'] = cls.conf['server']
        cls.servers = cls.init_from_conf(conf=cls.conf,
                                         single='server',
                                         multiple='servers',
                                         skip=skip,
                                         func=init_server_func)
        if cls.servers:
            cls.server = cls.servers.values()[0]

    @classmethod
    def init_comms(cls, init_comm_func=None, skip=None):
        """
        Initialize comms
        """
        if init_comm_func is None:
            init_comm_func = cls.init_comm
        cls.comms = cls.init_from_conf(conf=cls.conf,
                                       single='comm',
                                       multiple='comms',
                                       skip=skip,
                                       func=init_comm_func)
        if cls.comms:
            cls.comm = cls.comms.values()[0]

    @classmethod
    def init_schedulers(cls, init_sched_func=None, skip=None):
        """
        Initialize schedulers
        """
        if init_sched_func is None:
            init_sched_func = cls.init_scheduler
        cls.schedulers = cls.init_from_conf(conf=cls.conf,
                                            single='scheduler',
                                            multiple='schedulers',
                                            skip=skip,
                                            func=init_sched_func)
        if cls.schedulers:
            cls.scheduler = cls.schedulers.values()[0]

    @classmethod
    def init_moms(cls, init_mom_func=None, skip='nomom'):
        """
        Initialize moms
        """
        if init_mom_func is None:
            init_mom_func = cls.init_mom
        cls.moms = cls.init_from_conf(conf=cls.conf,
                                      single='mom',
                                      multiple='moms',
                                      skip=skip,
                                      func=init_mom_func)
        if cls.moms:
            cls.mom = cls.moms.values()[0]

    @classmethod
    def init_server(cls, hostname, pbsconf_file=None):
        """
        Initialize a server instance

        Define custom expect action to trigger a scheduling cycle when job
        is not in running state

        :returns: The server instance on success and None on failure
        """
        client = hostname
        client_conf = None
        if 'client' in cls.conf:
            _cl = cls.conf['client'].split('@')
            client = _cl[0]
            if len(_cl) > 1:
                client_conf = _cl[1]
        server = Server(hostname,
                        pbsconf_file=pbsconf_file,
                        client=client,
                        client_pbsconf_file=client_conf)
        server._conn_timeout = 0
        if cls.conf is not None:
            if 'mode' in cls.conf:
                if cls.conf['mode'] == 'cli':
                    server.set_op_mode(PTL_CLI)
            if 'conn_timeout' in cls.conf:
                conn_timeout = int(cls.conf['conn_timeout'])
                server.set_connect_timeout(conn_timeout)
        sched_action = ExpectAction('kicksched', True, JOB,
                                    cls.kicksched_action)
        server.add_expect_action(action=sched_action)
        return server

    @classmethod
    def init_comm(cls, hostname, pbsconf_file=None, server=None):
        """
        Initialize a Comm instance associated to the given hostname.

        This method must be called after init_server

        :param hostname: The host on which the Comm is running
        :type hostname: str
        :param pbsconf_file: Optional path to an alternate pbs config file
        :type pbsconf_file: str or None
        :returns: The instantiated Comm upon success and None on failure.
        :param server: The server name associated to the Comm
        :type server: str
        Return the instantiated Comm upon success and None on failure.
        """
        try:
            server = cls.servers[server]
        except:
            server = None
        return Comm(hostname, pbsconf_file=pbsconf_file, server=server)

    @classmethod
    def init_scheduler(cls, hostname, pbsconf_file=None, server=None):
        """
        Initialize a Scheduler instance associated to the given server.
        This method must be called after ``init_server``

        :param server: The server name associated to the scheduler
        :type server: str
        :param pbsconf_file: Optional path to an alternate config file
        :type pbsconf_file: str or None
        :param hostname: The host on which Sched is running
        :type hostname: str
        :returns: The instantiated scheduler upon success and None on failure
        """
        try:
            server = cls.servers[server]
        except:
            server = None
        return Scheduler(hostname=hostname,
                         server=server,
                         pbsconf_file=pbsconf_file)

    @classmethod
    def init_mom(cls, hostname, pbsconf_file=None, server=None):
        """
        Initialize a ``MoM`` instance associated to the given hostname.

        This method must be called after ``init_server``

        :param hostname: The host on which the MoM is running
        :type hostname: str
        :param pbsconf_file: Optional path to an alternate pbs config file
        :type pbsconf_file: str or None
        :returns: The instantiated MoM upon success and None on failure.
        """
        try:
            server = cls.servers[server]
        except:
            server = None
        return MoM(hostname, pbsconf_file=pbsconf_file, server=server)

    def init_proc_mon(self):
        """
        Initialize process monitoring when requested
        """
        if 'procmon' in self.conf:
            _proc_mon = []
            for p in self.conf['procmon'].split(':'):
                _proc_mon += ['.*' + p + '.*']
            if _proc_mon:
                if 'procmon-freq' in self.conf:
                    freq = int(self.conf['procmon-freq'])
                else:
                    freq = 10
                self.start_proc_monitor(name='|'.join(_proc_mon),
                                        regexp=True,
                                        frequency=freq)
                self._process_monitoring = True

    def revert_servers(self, force=False):
        """
        Revert the values set for servers
        """
        for server in self.servers.values():
            self.revert_server(server, force)

    def revert_comms(self, force=False):
        """
        Revert the values set for comms
        """
        for comm in self.comms.values():
            self.revert_comm(comm, force)

    def revert_schedulers(self, force=False):
        """
        Revert the values set for schedulers
        """
        for sched in self.schedulers.values():
            self.revert_scheduler(sched, force)

    def revert_moms(self, force=False):
        """
        Revert the values set for moms
        """
        for mom in self.moms.values():
            self.revert_mom(mom, force)

    def revert_server(self, server, force=False):
        """
        Revert the values set for server
        """
        rv = server.isUp()
        if not rv:
            self.logger.error('server ' + server.hostname + ' is down')
            server.start()
            msg = 'Failed to restart server ' + server.hostname
            self.assertTrue(server.isUp(), msg)
        server_stat = server.status(SERVER)[0]
        current_user = pwd.getpwuid(os.getuid())[0]
        try:
            # remove current user's entry from managers list (if any)
            a = {ATTR_managers: (DECR, current_user + '@*')}
            server.manager(MGR_CMD_SET, SERVER, a, sudo=True)
        except:
            pass
        a = {ATTR_managers: (INCR, current_user + '@*')}
        server.manager(MGR_CMD_SET, SERVER, a, sudo=True)
        if ((self.revert_to_defaults and self.server_revert_to_defaults)
                or force):
            server.revert_to_defaults(reverthooks=self.revert_hooks,
                                      delhooks=self.del_hooks,
                                      revertqueues=self.revert_queues,
                                      delqueues=self.del_queues,
                                      revertresources=self.revert_resources,
                                      server_stat=server_stat)
        rv = self.is_server_licensed(server)
        _msg = 'No license found on server %s' % (server.shortname)
        self.assertTrue(rv, _msg)
        self.logger.info('server: %s licensed', server.hostname)

    def revert_comm(self, comm, force=False):
        """
        Revert the values set for comm
        """
        rv = comm.isUp()
        if not rv:
            self.logger.error('comm ' + comm.hostname + ' is down')
            comm.start()
            msg = 'Failed to restart comm ' + comm.hostname
            self.assertTrue(comm.isUp(), msg)

    def revert_scheduler(self, scheduler, force=False):
        """
        Revert the values set for scheduler
        """
        rv = scheduler.isUp()
        if not rv:
            self.logger.error('scheduler ' + scheduler.hostname + ' is down')
            scheduler.start()
            msg = 'Failed to restart scheduler ' + scheduler.hostname
            self.assertTrue(scheduler.isUp(), msg)
        if ((self.revert_to_defaults and self.sched_revert_to_defaults)
                or force):
            rv = scheduler.revert_to_defaults()
            _msg = 'Failed to revert sched %s' % (scheduler.hostname)
            self.assertTrue(rv, _msg)

    def revert_mom(self, mom, force=False):
        """
        Revert the values set for mom

        :param mom: the MoM object whose values are to be reverted
        :type mom: MoM object
        :param force: Option to reverse forcibly
        :type force: bool
        """
        rv = mom.isUp()
        if not rv:
            self.logger.error('mom ' + mom.hostname + ' is down')
            mom.start()
            msg = 'Failed to restart mom ' + mom.hostname
            self.assertTrue(mom.isUp(), msg)
        mom.pbs_version()
        if ((self.revert_to_defaults and self.mom_revert_to_defaults)
                or force):
            rv = mom.revert_to_defaults(delvnodedefs=self.del_vnodes)
            _msg = 'Failed to revert mom %s' % (mom.hostname)
            self.assertTrue(rv, _msg)
            if 'clienthost' in self.conf:
                mom.add_config({'$clienthost': self.conf['clienthost']})
            a = {'state': 'free', 'resources_available.ncpus': (GE, 1)}
            nodes = self.server.counter(NODE,
                                        a,
                                        attrop=PTL_AND,
                                        level=logging.DEBUG)
            if not nodes:
                try:
                    self.server.manager(MGR_CMD_DELETE, NODE, None, '')
                except:
                    pass
                mom.delete_vnode_defs()
                mom.delete_vnodes()
                mom.restart()
                self.logger.info('server: no nodes defined, creating one')
                self.server.manager(MGR_CMD_CREATE, NODE, None, mom.shortname)
        name = mom.shortname
        try:
            self.server.status(NODE, id=name)
        except PbsStatusError:
            # server doesn't have node with shortname
            # check with hostname
            name = mom.hostname
            try:
                self.server.status(NODE, id=name)
            except PbsStatusError:
                # server doesn't have node for this mom yet
                # so create with shortname
                name = mom.shortname
                self.server.manager(MGR_CMD_CREATE, NODE, None, mom.shortname)
        self.server.expect(NODE, {ATTR_NODE_state: 'free'},
                           id=name,
                           interval=1)
        return mom

    def analyze_logs(self):
        """
        analyze accounting and scheduler logs from time test was started
        until it finished
        """
        pla = PBSLogAnalyzer()
        self.metrics_data = pla.analyze_logs(serverlog=self.server.logfile,
                                             schedlog=self.scheduler.logfile,
                                             momlog=self.mom.logfile,
                                             acctlog=self.server.acctlogfile,
                                             start=self.server.ctime,
                                             end=int(time.time()))

    def start_proc_monitor(self, name=None, regexp=False, frequency=60):
        """
        Start the process monitoring

        :param name: Process name
        :type name: str or None
        :param regexp: Regular expression to match
        :type regexp: bool
        :param frequency: Frequency of monitoring
        :type frequency: int
        """
        if self._procmon is not None:
            self.logger.info('A process monitor is already instantiated')
            return
        self.logger.info('starting process monitoring of ' + name + ' every ' +
                         str(frequency) + 'seconds')
        self._procmon = ProcMonitor(name=name,
                                    regexp=regexp,
                                    frequency=frequency)
        self._procmon.start()

    def stop_proc_monitor(self):
        """
        Stop the process monitoring
        """
        if not self._process_monitoring:
            return
        self.logger.info('stopping process monitoring')
        self._procmon.stop()
        self.metrics_data['procs'] = self._procmon.db_proc_info
        self._process_monitoring = False

    def skipTest(self, reason=None):
        """
        Skip Test

        :param reason: message to indicate why test is skipped
        :type reason: str or None
        """
        if reason:
            self.logger.warning('test skipped: ' + reason)
        else:
            reason = 'unknown'
        raise SkipTest(reason)

    skip_test = skipTest

    @classmethod
    def log_enter_teardown(cls, iscls=False):
        _m = ' Entered ' + cls.__name__ + ' tearDown'
        if iscls:
            _m += 'Class'
        _m_len = len(_m)
        cls.logger.info('=' * _m_len)
        cls.logger.info(_m)
        cls.logger.info('=' * _m_len)

    @classmethod
    def log_end_teardown(cls, iscls=False):
        _m = 'Completed ' + cls.__name__ + ' tearDown'
        if iscls:
            _m += 'Class'
        _m_len = len(_m)
        cls.logger.info('=' * _m_len)
        cls.logger.info(_m)
        cls.logger.info('=' * _m_len)

    def tearDown(self):
        """
        verify that ``server`` and ``scheduler`` are up
        clean up jobs and reservations
        """
        if 'skip-teardown' in self.conf:
            return
        self.log_enter_teardown()
        self.stop_proc_monitor()
        self.log_end_teardown()

    @classmethod
    def tearDownClass(cls):
        cls._testMethodName = 'tearDownClass'
Ejemplo n.º 7
0
class PBSTestSuite(unittest.TestCase):

    """
    Generic setup, teardown, and logging functions to be used as parent class
    for most tests.
    Class instantiates:
    server object connected to localhost
    scheduler objected connected to localhost
    mom object connected to localhost

    Custom parameters:

    server: The hostname on which the PBS server/scheduler are running

    mom: The hostname on which the PBS MoM is running

    servers: Colon-separated list of hostnames hosting a PBS server.
    Servers are then accessible as a dictionary in the instance variable
    servers.

    client: For CLI mode only, name of the host on which the PBS client
    commands are to be run from. Format is <host>@<path-to-config-file>

    moms: Colon-separated list of hostnames hosting a PBS MoM. MoMs are made
    accessible as a dictionary in the instance variable moms.

    comms: Colon-separated list of hostnames hosting a PBS Comm. Comms are made
    accessible as a dictionary in the instance variable comms.

    nomom=<host1>:<host2>...: expect no MoM on given set of hosts

    mode: Sets mode of operation to PBS server. Can be either 'cli' or 'api'.
    Defaults to API behavior.

    conn_timeout: set a timeout in seconds after which a pbs_connect IFL call
    is refreshed (i.e., disconnected)

    skip-setup: Bypasses setUp of PBSTestSuite (not custom ones)

    skip-teardown: Bypasses tearDown of PBSTestSuite (not custom ones)

    procinfo: Enables process monitoring thread, logged into ptl_proc_info test
    metrics. The value can be set to _all_ to monitor all PBS processes,
    including pbs_server, pbs_sched, pbs_mom, or a process defined by name.

    revert-to-defaults=<True|False>: if False, will not revert to defaults.
    True by default.

    revert-hooks=<True|False>: if False, do not revert hooks to defaults.
    Defaults to True. revert-to-defaults set to False overrides this setting.

    del-hooks=<True|False>: If False, do not delete hooks. Defaults to False.
    revert-to-defaults set to False overrides this setting.

    revert-queues=<True|False>: If False, do not revert queues to defaults.
    Defaults to True. revert-to-defaults set to False overrides this setting.

    revert-resources=<True|False>: If False, do not revert resources to
    defaults. Defaults to True. revert-to-defaults set to False overrides this
    setting.

    del-queues=<True|False>: If False, do not delete queues. Defaults to False.
    revert-to-defaults set to False overrides this setting.

    del-vnodes=<True|False>: If False, do not delete vnodes on MoM instances.
    Defaults to True.

    server-revert-to-defaults=<True|False>: if False, don't revert Server to
    defaults

    comm-revert-to-defaults=<True|False>: if False, don't revert Comm to
    defaults

    mom-revert-to-defaults=<True|False>: if False, don't revert MoM to defaults

    sched-revert-to-defaults=<True|False>: if False, don't revert Scheduler to
    defaults

    procmon: Enables process monitoring. Multiple values must be colon
    separated. For example to monitor server, sched, and mom use
    procmon=pbs_server:pbs_sched:pbs_mom

    procmon-freq: Sets a polling frequency for the process monitoring tool.
    Defaults to 10 seconds.

    test-users: colon-separated list of users to use as test users. The users
    specified override the default users in the order in which they appear in
    the PBS_USERS list.

    data-users: colon-separated list of data users.

    oper-users: colon-separated list of operator users.

    mgr-users: colon-separated list of manager users.

    root-users: colon-separated list of root users.

    build-users: colon-separated list of build users.

    clienthost: the hostnames to set in the MoM config file
    """

    logger = logging.getLogger(__name__)
    metrics_data = {}
    conf = {}
    param = None
    du = DshUtils()
    _procmon = None
    _process_monitoring = False
    revert_to_defaults = True
    server_revert_to_defaults = True
    mom_revert_to_defaults = True
    sched_revert_to_defaults = True
    revert_queues = True
    revert_resources = True
    revert_hooks = True
    del_hooks = True
    del_queues = True
    del_vnodes = True
    server = None
    scheduler = None
    mom = None
    comm = None
    servers = None
    schedulers = None
    moms = None
    comms = None

    @classmethod
    def setUpClass(cls):
        cls.log_enter_setup(True)
        cls._testMethodName = "setUpClass"
        cls.parse_param()
        cls.init_param()
        cls.check_users_exist()
        cls.init_servers()
        cls.init_comms()
        cls.init_schedulers()
        cls.init_moms()
        cls.log_end_setup(True)

    def setUp(self):
        if "skip-setup" in self.conf:
            return
        self.log_enter_setup()
        self.init_proc_mon()
        self.revert_servers()
        self.revert_comms()
        self.revert_schedulers()
        self.revert_moms()
        self.log_end_setup()

    @classmethod
    def log_enter_setup(cls, iscls=False):
        _m = " Entered " + cls.__name__ + " setUp"
        if iscls:
            _m += "Class"
        _m_len = len(_m)
        cls.logger.info("=" * _m_len)
        cls.logger.info(_m)
        cls.logger.info("=" * _m_len)

    @classmethod
    def log_end_setup(cls, iscls=False):
        _m = "Completed " + cls.__name__ + " setUp"
        if iscls:
            _m += "Class"
        _m_len = len(_m)
        cls.logger.info("=" * _m_len)
        cls.logger.info(_m)
        cls.logger.info("=" * _m_len)

    @classmethod
    def _validate_param(cls, pname):
        """
        Check if parameter was enabled at the command-line

        pname - parameter name

        pvar - class variable to set according to command-line setting
        """
        if pname not in cls.conf:
            return
        if cls.conf[pname] in PTL_TRUE:
            setattr(cls, pname.replace("-", "_"), True)
        else:
            setattr(cls, pname.replace("-", "_"), False)

    @classmethod
    def _set_user(cls, name, user_list):
        if name in cls.conf:
            for idx, u in enumerate(cls.conf[name].split(":")):
                user_list[idx].__init__(u)

    @classmethod
    def check_users_exist(cls):
        testusersexist = True
        for u in [TEST_USER, TEST_USER1, TEST_USER2, TEST_USER3]:
            rv = cls.du.check_user_exists(str(u))
            if not rv:
                _msg = "User " + str(u) + " does not exist!"
                raise setUpClassError(_msg)
        return testusersexist

    @classmethod
    def kicksched_action(cls, server, obj_type, *args, **kwargs):
        """
        custom scheduler action to kick a scheduling cycle when expectig a job
        state change
        """
        if server is None:
            cls.logger.error("no server defined for custom action")
            return
        if obj_type == JOB:
            if ("scheduling" in server.attributes) and (server.attributes["scheduling"] != "False"):
                server.manager(MGR_CMD_SET, MGR_OBJ_SERVER, {"scheduling": "True"}, level=logging.DEBUG)

    @classmethod
    def parse_param(cls):
        """
        get test configuration parameters as a comma-separated list of
        attributes.

        Attributes may be '=' separated key value pairs or standalone entries.

        Multi-property attributes are colon-delimited.
        """
        if cls.param is None:
            return
        for h in cls.param.split(","):
            if "=" in h:
                k, v = h.split("=")
                cls.conf[k.strip()] = v.strip()
            else:
                cls.conf[h.strip()] = ""
        if ("clienthost" in cls.conf) and not isinstance(cls.conf["clienthost"], list):
            cls.conf["clienthost"] = cls.conf["clienthost"].split(":")
        users_map = [
            ("test-users", PBS_USERS),
            ("oper-users", PBS_OPER_USERS),
            ("mgr-users", PBS_MGR_USERS),
            ("data-users", PBS_DATA_USERS),
            ("root-users", PBS_ROOT_USERS),
            ("build-users", PBS_BUILD_USERS),
        ]
        for k, v in users_map:
            cls._set_user(k, v)

    @classmethod
    def init_param(cls):
        cls._validate_param("revert-to-defaults")
        cls._validate_param("server-revert-to-defaults")
        cls._validate_param("comm-revert-to-defaults")
        cls._validate_param("mom-revert-to-defaults")
        cls._validate_param("sched-revert-to-defaults")
        cls._validate_param("del-hooks")
        cls._validate_param("revert-hooks")
        cls._validate_param("del-queues")
        cls._validate_param("del-vnodes")
        cls._validate_param("revert-queues")
        cls._validate_param("revert-resources")

    @classmethod
    def is_server_licensed(cls, server):
        for i in range(0, 10, 1):
            lic = server.status(SERVER, "license_count", level=logging.INFOCLI)
            if lic and "license_count" in lic[0]:
                lic = PbsTypeLicenseCount(lic[0]["license_count"])
                if int(lic["Avail_Sockets"]) > 0:
                    return True
                elif int(lic["Avail_Global"]) > 0:
                    return True
                elif int(lic["Avail_Local"]) > 0:
                    return True
            time.sleep(i)
        return False

    @classmethod
    def init_from_conf(cls, conf, single=None, multiple=None, skip=None, func=None):
        """
        Helper method to parse test parameters for mom/server/scheduler
        instances.

        The supported format of each service request is:

        hostname@configuration/path

        For example:

        pbs_benchpress -p server=remote@/etc/pbs.conf.12.0

        initializes a remote server instance that is configured according to
        the remote file /etc/pbs.conf.12.0
        """
        endpoints = []
        if (multiple in conf) and (conf[multiple] is not None):
            __objs = conf[multiple].split(":")
            for _m in __objs:
                tmp = _m.split("@")
                if len(tmp) == 2:
                    endpoints.append(tuple(tmp))
                elif len(tmp) == 1:
                    endpoints.append((tmp[0], None))
        elif (single in conf) and (conf[single] is not None):
            tmp = conf[single].split("@")
            if len(tmp) == 2:
                endpoints.append(tuple(tmp))
            elif len(tmp) == 1:
                endpoints.append((tmp[0], None))
        else:
            endpoints = [(socket.gethostname(), None)]
        objs = PBSServiceInstanceWrapper()
        for name, objconf in endpoints:
            if (skip is not None) and (skip in conf) and ((name in conf[skip]) or (conf[skip] in name)):
                continue
            if objconf is not None:
                n = name + "@" + objconf
            else:
                n = name
            objs[n] = func(name, pbsconf_file=objconf)
            if objs[n] is None:
                _msg = "Failed %s(%s, %s)" % (func.__name__, name, objconf)
                raise setUpClassError(_msg)
        return objs

    @classmethod
    def init_servers(cls, init_server_func=None, skip=None):
        if init_server_func is None:
            init_server_func = cls.init_server
        if "servers" in cls.conf:
            if "comms" not in cls.conf:
                cls.conf["comms"] = cls.conf["servers"]
            if "schedulers" not in cls.conf:
                cls.conf["schedulers"] = cls.conf["servers"]
            if "moms" not in cls.conf:
                cls.conf["moms"] = cls.conf["servers"]
        if "server" in cls.conf:
            if "comm" not in cls.conf:
                cls.conf["comm"] = cls.conf["server"]
            if "scheduler" not in cls.conf:
                cls.conf["scheduler"] = cls.conf["server"]
            if "mom" not in cls.conf:
                cls.conf["mom"] = cls.conf["server"]
        cls.servers = cls.init_from_conf(
            conf=cls.conf, single="server", multiple="servers", skip=skip, func=init_server_func
        )
        if cls.servers:
            cls.server = cls.servers.values()[0]

    @classmethod
    def init_comms(cls, init_comm_func=None, skip=None):
        if init_comm_func is None:
            init_comm_func = cls.init_comm
        cls.comms = cls.init_from_conf(conf=cls.conf, single="comm", multiple="comms", skip=skip, func=init_comm_func)
        if cls.comms:
            cls.comm = cls.comms.values()[0]

    @classmethod
    def init_schedulers(cls, init_sched_func=None, skip=None):
        if init_sched_func is None:
            init_sched_func = cls.init_scheduler
        cls.schedulers = cls.init_from_conf(
            conf=cls.conf, single="scheduler", multiple="schedulers", skip=skip, func=init_sched_func
        )
        if cls.schedulers:
            cls.scheduler = cls.schedulers.values()[0]

    @classmethod
    def init_moms(cls, init_mom_func=None, skip="nomom"):
        if init_mom_func is None:
            init_mom_func = cls.init_mom
        cls.moms = cls.init_from_conf(conf=cls.conf, single="mom", multiple="moms", skip=skip, func=init_mom_func)
        if cls.moms:
            cls.mom = cls.moms.values()[0]

    @classmethod
    def init_server(cls, hostname, pbsconf_file=None):
        """
        Initialize a server instance

        Define custom expect action to trigger a scheduling cycle when job
        is not in running state

        Return the server instance on success and None on failure
        """
        client = hostname
        client_conf = None
        if "client" in cls.conf:
            _cl = cls.conf["client"].split("@")
            client = _cl[0]
            if len(_cl) > 1:
                client_conf = _cl[1]
        server = Server(hostname, pbsconf_file=pbsconf_file, client=client, client_pbsconf_file=client_conf)
        server._conn_timeout = 0
        if cls.conf is not None:
            if "mode" in cls.conf:
                if cls.conf["mode"] == "cli":
                    server.set_op_mode(PTL_CLI)
            if "conn_timeout" in cls.conf:
                conn_timeout = int(cls.conf["conn_timeout"])
                server.set_connect_timeout(conn_timeout)
        sched_action = ExpectAction("kicksched", True, JOB, cls.kicksched_action)
        server.add_expect_action(action=sched_action)
        return server

    @classmethod
    def init_comm(cls, hostname, pbsconf_file=None):
        """
        Initialize a Comm instance associated to the given hostname.

        This method must be called after init_server

        hostname - The host on which the Comm is running

        pbsconf_file - Optional path to an alternate pbs config file

        Return the instantiated Comm upon success and None on failure.
        """
        return Comm(hostname, pbsconf_file=pbsconf_file)

    @classmethod
    def init_scheduler(cls, server, pbsconf_file=None):
        """
        Initialize a Scheduler instance associated to the given server.
        This method must be called after init_server

        server - The server name associated to the scheduler

        pbsconf_file - Optional path to an alternate config file

        Return the instantiated scheduler upon success and None on failure
        """
        try:
            server = cls.servers[server]
        except:
            server = None
        return Scheduler(server=server, pbsconf_file=pbsconf_file)

    @classmethod
    def init_mom(cls, hostname, pbsconf_file=None, server=None):
        """
        Initialize a MoM instance associated to the given hostname.

        This method must be called after init_server

        hostname - The host on which the MoM is running

        pbsconf_file - Optional path to an alternate pbs config file

        server - The server name associated to the mom

        Return the instantiated MoM upon success and None on failure.
        """
        try:
            server = cls.servers[server]
        except:
            server = None
        return MoM(hostname, pbsconf_file=pbsconf_file, server=server)

    def init_proc_mon(self):
        """
        Initialize process monitoring when requested
        """
        if "procmon" in self.conf:
            _proc_mon = []
            for p in self.conf["procmon"].split(":"):
                _proc_mon += [".*" + p + ".*"]
            if _proc_mon:
                if "procmon-freq" in self.conf:
                    freq = int(self.conf["procmon-freq"])
                else:
                    freq = 10
                self.start_proc_monitor(name="|".join(_proc_mon), regexp=True, frequency=freq)
                self._process_monitoring = True

    def revert_servers(self, force=False):
        for server in self.servers.values():
            self.revert_server(server, force)

    def revert_comms(self, force=False):
        for comm in self.comms.values():
            self.revert_comm(comm, force)

    def revert_schedulers(self, force=False):
        for sched in self.schedulers.values():
            self.revert_scheduler(sched, force)

    def revert_moms(self, force=False):
        for mom in self.moms.values():
            self.revert_mom(mom, force)

    def revert_server(self, server, force=False):
        rv = server.isUp()
        if not rv:
            self.logger.error("server " + server.hostname + " is down")
            server.start()
            msg = "Failed to restart server " + server.hostname
            self.assertTrue(server.isUp(), msg)
        server_stat = server.status(SERVER)[0]
        current_user = pwd.getpwuid(os.getuid())[0]
        try:
            # remove current user's entry from managers list (if any)
            a = {ATTR_managers: (DECR, current_user + "@*")}
            server.manager(MGR_CMD_SET, SERVER, a, sudo=True)
        except:
            pass
        a = {ATTR_managers: (INCR, current_user + "@*")}
        server.manager(MGR_CMD_SET, SERVER, a, sudo=True)
        if (self.revert_to_defaults and self.server_revert_to_defaults) or force:
            server.revert_to_defaults(
                reverthooks=self.revert_hooks,
                delhooks=self.del_hooks,
                revertqueues=self.revert_queues,
                delqueues=self.del_queues,
                revertresources=self.revert_resources,
                server_stat=server_stat,
            )
        rv = self.is_server_licensed(server)
        _msg = "No license found on server %s" % (server.shortname)
        self.assertTrue(rv, _msg)
        self.logger.info("server: %s licensed", server.hostname)

    def revert_comm(self, comm, force=False):
        rv = comm.isUp()
        if not rv:
            self.logger.error("comm " + comm.hostname + " is down")
            comm.start()
            msg = "Failed to restart comm " + comm.hostname
            self.assertTrue(comm.isUp(), msg)

    def revert_scheduler(self, scheduler, force=False):
        rv = scheduler.isUp()
        if not rv:
            self.logger.error("scheduler " + scheduler.hostname + " is down")
            scheduler.start()
            msg = "Failed to restart scheduler " + scheduler.hostname
            self.assertTrue(scheduler.isUp(), msg)
        if (self.revert_to_defaults and self.sched_revert_to_defaults) or force:
            rv = scheduler.revert_to_defaults()
            _msg = "Failed to revert sched %s" % (scheduler.hostname)
            self.assertTrue(rv, _msg)

    def revert_mom(self, mom, force=False):
        # below call is a Noop if no switching is needed
        mom.switch_to_standard_mom()
        rv = mom.isUp()
        if not rv:
            self.logger.error("mom " + mom.hostname + " is down")
            mom.start()
            msg = "Failed to restart mom " + mom.hostname
            self.assertTrue(mom.isUp(), msg)
        mom.pbs_version()
        if (self.revert_to_defaults and self.mom_revert_to_defaults) or force:
            rv = mom.revert_to_defaults(delvnodedefs=self.del_vnodes)
            _msg = "Failed to revert mom %s" % (mom.hostname)
            self.assertTrue(rv, _msg)
            if "clienthost" in self.conf:
                mom.add_config({"$clienthost": self.conf["clienthost"]})
            a = {"state": "free", "resources_available.ncpus": (GE, 1)}
            nodes = self.server.counter(NODE, a, attrop=PTL_AND, level=logging.DEBUG)
            if not nodes:
                try:
                    self.server.manager(MGR_CMD_DELETE, NODE, None, "")
                except:
                    pass
                mom.delete_vnode_defs()
                mom.restart()
                self.logger.info("server: no nodes defined, creating one")
                self.server.manager(MGR_CMD_CREATE, NODE, None, mom.shortname)
                a = {"resources_available.ncpus": 8}
                self.server.manager(MGR_CMD_SET, NODE, a, mom.shortname, expect=True)
        name = mom.shortname
        try:
            self.server.status(NODE, id=name)
        except PbsStatusError:
            # server doesn't have node with shortname
            # check with hostname
            name = mom.hostname
            try:
                self.server.status(NODE, id=name)
            except PbsStatusError:
                # server doesn't have node for this mom yet
                # so create with shortname
                name = mom.shortname
                self.server.manager(MGR_CMD_CREATE, NODE, None, mom.shortname)
        self.server.expect(NODE, {ATTR_NODE_state: "free"}, id=name, interval=1)
        return mom

    def analyze_logs(self):
        """
        analyze accounting and scheduler logs from time test was started until
        it finished
        """
        pla = PBSLogAnalyzer()
        self.metrics_data = pla.analyze_logs(
            serverlog=self.server.logfile,
            schedlog=self.scheduler.logfile,
            momlog=self.mom.logfile,
            acctlog=self.server.acctlogfile,
            start=self.server.ctime,
            end=int(time.time()),
        )

    def start_proc_monitor(self, name=None, regexp=False, frequency=60):
        if self._procmon is not None:
            self.logger.info("A process monitor is already instantiated")
            return
        self.logger.info("starting process monitoring of " + name + " every " + str(frequency) + "seconds")
        self._procmon = ProcMonitor(name=name, regexp=regexp, frequency=frequency)
        self._procmon.start()

    def stop_proc_monitor(self):
        if not self._process_monitoring:
            return
        self.logger.info("stopping process monitoring")
        self._procmon.stop()
        self.metrics_data["procs"] = self._procmon.db_proc_info
        self._process_monitoring = False

    def skipTest(self, reason=None):
        """
        Skip Test
        reason - message to indicate why test is skipped
        """
        if reason:
            self.logger.warning("test skipped: " + reason)
        else:
            reason = "unknown"
        raise SkipTest(reason)

    skip_test = skipTest

    @classmethod
    def log_enter_teardown(cls, iscls=False):
        _m = " Entered " + cls.__name__ + " tearDown"
        if iscls:
            _m += "Class"
        _m_len = len(_m)
        cls.logger.info("=" * _m_len)
        cls.logger.info(_m)
        cls.logger.info("=" * _m_len)

    @classmethod
    def log_end_teardown(cls, iscls=False):
        _m = "Completed " + cls.__name__ + " tearDown"
        if iscls:
            _m += "Class"
        _m_len = len(_m)
        cls.logger.info("=" * _m_len)
        cls.logger.info(_m)
        cls.logger.info("=" * _m_len)

    def tearDown(self):
        """
        verify that server and scheduler are up
        clean up jobs and reservations
        """
        if "skip-teardown" in self.conf:
            return
        self.log_enter_teardown()
        self.stop_proc_monitor()
        self.log_end_teardown()

    @classmethod
    def tearDownClass(cls):
        cls._testMethodName = "tearDownClass"