def test_load__two_extensions(self, mock_listdir, mock_isfile, mock_open, mock_logger): """ verify that FsPlugInCollection works with multiple extensions """ mock_listdir.return_value = ["foo.txt", "bar.txt.in"] mock_isfile.return_value = True def fake_open(path, encoding=None, mode=None): m = mock.MagicMock(name='opened file {!r}'.format(path)) m.read.return_value = "text" m.__enter__.return_value = m return m mock_open.side_effect = fake_open # Create a collection that looks for both extensions col = FsPlugInCollection([self._P1], (".txt", ".txt.in")) # Load everything col.load() # Ensure that we actually tried to look at the filesystem self.assertEqual( mock_listdir.call_args_list, [ ((self._P1, ), {}), ]) # Ensure that we actually tried to check if things are files self.assertEqual( mock_isfile.call_args_list, [ ((os.path.join(self._P1, 'foo.txt'),), {}), ((os.path.join(self._P1, 'bar.txt.in'),), {}), ]) # Ensure that we actually tried to open some files self.assertEqual( mock_open.call_args_list, [ ((os.path.join(self._P1, 'bar.txt.in'),), {'encoding': 'UTF-8'}), ((os.path.join(self._P1, 'foo.txt'),), {'encoding': 'UTF-8'}), ]) # Ensure that no exception was logged mock_logger.error.assert_not_called() # Ensure that everything was okay self.assertEqual(col.problem_list, []) # Ensure that both files got added self.assertEqual( col.get_by_name( os.path.join(self._P1, "foo.txt") ).plugin_object, "text") self.assertEqual( col.get_by_name( os.path.join(self._P1, "bar.txt.in") ).plugin_object, "text")
def __init__(self, base_dir, name, version, description, secure, gettext_domain=None): """ Initialize the provider with the associated base directory. All of the typical v1 provider data is relative to this directory. It can be customized by subclassing and overriding the particular methods of the IProviderBackend1 class but that should not be necessary in normal operation. """ self._base_dir = base_dir self._name = name self._version = version self._description = description self._secure = secure self._gettext_domain = gettext_domain self._whitelist_collection = FsPlugInCollection( [self.whitelists_dir], ext=".whitelist", wrapper=WhiteListPlugIn) self._job_collection = FsPlugInCollection( [self.jobs_dir], ext=(".txt", ".txt.in"), wrapper=JobDefinitionPlugIn, provider=self)
def test_load__two_extensions(self, mock_listdir, mock_isfile, mock_open, mock_logger): """ verify that FsPlugInCollection works with multiple extensions """ mock_listdir.return_value = ["foo.txt", "bar.txt.in"] mock_isfile.return_value = True def fake_open(path, encoding=None, mode=None): m = mock.MagicMock(name='opened file {!r}'.format(path)) m.read.return_value = "text" m.__enter__.return_value = m return m mock_open.side_effect = fake_open # Create a collection that looks for both extensions col = FsPlugInCollection([self._P1], (".txt", ".txt.in")) # Load everything col.load() # Ensure that we actually tried to look at the filesystem self.assertEqual(mock_listdir.call_args_list, [ ((self._P1, ), {}), ]) # Ensure that we actually tried to check if things are files self.assertEqual(mock_isfile.call_args_list, [ ((os.path.join(self._P1, 'foo.txt'), ), {}), ((os.path.join(self._P1, 'bar.txt.in'), ), {}), ]) # Ensure that we actually tried to open some files self.assertEqual(mock_open.call_args_list, [ ((os.path.join(self._P1, 'bar.txt.in'), ), { 'encoding': 'UTF-8' }), ((os.path.join(self._P1, 'foo.txt'), ), { 'encoding': 'UTF-8' }), ]) # Ensure that no exception was logged self.assertEqual(mock_logger.error.mock_calls, []) # Ensure that everything was okay self.assertEqual(col.problem_list, []) # Ensure that both files got added self.assertEqual( col.get_by_name(os.path.join(self._P1, "foo.txt")).plugin_object, "text") self.assertEqual( col.get_by_name(os.path.join(self._P1, "bar.txt.in")).plugin_object, "text")
def setUp(self): # Create a collection self.col = FsPlugInCollection(self._DIR_LIST, self._EXT)
class FsPlugInCollectionTests(TestCase): _P1 = "/system/providers" _P2 = "home/user/.providers" _DIR_LIST = [_P1, _P2] _EXT = ".plugin" def setUp(self): # Create a collection self.col = FsPlugInCollection(self._DIR_LIST, self._EXT) def test_path_is_set(self): # Ensure that path was saved self.assertEqual(self.col._dir_list, self._DIR_LIST) def test_ext_is_set(self): # Ensure that ext was saved self.assertEqual(self.col._ext, self._EXT) def test_plugins_are_empty(self): # Ensure that plugins start out empty self.assertEqual(len(self.col._plugins), 0) def test_initial_loaded_flag(self): # Ensure that 'loaded' flag is false self.assertFalse(self.col._loaded) def test_default_wrapper(self): # Ensure that the wrapper is :class:`PlugIn` self.assertEqual(self.col._wrapper, PlugIn) @mock.patch('plainbox.impl.secure.plugins.logger') @mock.patch('builtins.open') @mock.patch('os.path.isfile') @mock.patch('os.listdir') def test_load(self, mock_listdir, mock_isfile, mock_open, mock_logger): # Mock a bit of filesystem access methods to make some plugins show up def fake_listdir(path): if path == self._P1: return [ # A regular plugin 'foo.plugin', # Another regular plugin 'bar.plugin', # Unrelated file, not a plugin 'unrelated.txt', # A directory that looks like a plugin 'dir.bad.plugin', # A plugin without read permissions 'noperm.plugin' ] else: raise OSError("There is nothing in {}".format(path)) def fake_isfile(path): return not os.path.basename(path).startswith('dir.') def fake_open(path, encoding=None, mode=None): m = mock.MagicMock(name='opened file {!r}'.format(path)) m.__enter__.return_value = m if path == os.path.join(self._P1, 'foo.plugin'): m.read.return_value = "foo" return m elif path == os.path.join(self._P1, 'bar.plugin'): m.read.return_value = "bar" return m elif path == os.path.join(self._P1, 'noperm.plugin'): raise OSError("You cannot open this file") else: raise IOError("Unexpected file: {}".format(path)) mock_listdir.side_effect = fake_listdir mock_isfile.side_effect = fake_isfile mock_open.side_effect = fake_open # Load all plugins now self.col.load() # And 'again', just to ensure we're doing the IO only once self.col.load() # Ensure that we actually tried to look at the filesytstem self.assertEqual(mock_listdir.call_args_list, [((self._P1, ), {}), ((self._P2, ), {})]) # Ensure that we actually tried to check if things are files self.assertEqual(mock_isfile.call_args_list, [ ((os.path.join(self._P1, 'foo.plugin'), ), {}), ((os.path.join(self._P1, 'bar.plugin'), ), {}), ((os.path.join(self._P1, 'dir.bad.plugin'), ), {}), ((os.path.join(self._P1, 'noperm.plugin'), ), {}), ]) # Ensure that we actually tried to open some files self.assertEqual(mock_open.call_args_list, [ ((os.path.join(self._P1, 'bar.plugin'), ), { 'encoding': 'UTF-8' }), ((os.path.join(self._P1, 'foo.plugin'), ), { 'encoding': 'UTF-8' }), ((os.path.join(self._P1, 'noperm.plugin'), ), { 'encoding': 'UTF-8' }), ]) # Ensure that an exception was logged mock_logger.error.assert_called_with( 'Unable to load %r: %s', '/system/providers/noperm.plugin', 'You cannot open this file') # Ensure that all of the errors are collected # Using repr() since OSError seems hard to compare correctly self.assertEqual(repr(self.col.problem_list[0]), repr(OSError('You cannot open this file'))) @mock.patch('plainbox.impl.secure.plugins.logger') @mock.patch('builtins.open') @mock.patch('os.path.isfile') @mock.patch('os.listdir') def test_load__two_extensions(self, mock_listdir, mock_isfile, mock_open, mock_logger): """ verify that FsPlugInCollection works with multiple extensions """ mock_listdir.return_value = ["foo.txt", "bar.txt.in"] mock_isfile.return_value = True def fake_open(path, encoding=None, mode=None): m = mock.MagicMock(name='opened file {!r}'.format(path)) m.read.return_value = "text" m.__enter__.return_value = m return m mock_open.side_effect = fake_open # Create a collection that looks for both extensions col = FsPlugInCollection([self._P1], (".txt", ".txt.in")) # Load everything col.load() # Ensure that we actually tried to look at the filesystem self.assertEqual(mock_listdir.call_args_list, [ ((self._P1, ), {}), ]) # Ensure that we actually tried to check if things are files self.assertEqual(mock_isfile.call_args_list, [ ((os.path.join(self._P1, 'foo.txt'), ), {}), ((os.path.join(self._P1, 'bar.txt.in'), ), {}), ]) # Ensure that we actually tried to open some files self.assertEqual(mock_open.call_args_list, [ ((os.path.join(self._P1, 'bar.txt.in'), ), { 'encoding': 'UTF-8' }), ((os.path.join(self._P1, 'foo.txt'), ), { 'encoding': 'UTF-8' }), ]) # Ensure that no exception was logged self.assertEqual(mock_logger.error.mock_calls, []) # Ensure that everything was okay self.assertEqual(col.problem_list, []) # Ensure that both files got added self.assertEqual( col.get_by_name(os.path.join(self._P1, "foo.txt")).plugin_object, "text") self.assertEqual( col.get_by_name(os.path.join(self._P1, "bar.txt.in")).plugin_object, "text")
def __init__(self, name, version, description, secure, gettext_domain, jobs_dir, whitelists_dir, data_dir, bin_dir, locale_dir, base_dir): """ Initialize a provider with a set of meta-data and directories. :param name: provider name / ID :param version: provider version :param description: provider version This is the untranslated version of this field. Implementations may obtain the localized version based on the gettext_domain property. :param secure: secure bit When True jobs from this provider should be available via the trusted launcher mechanism. It should be set to True for system-wide installed providers. :param gettext_domain: gettext domain that contains translations for this provider :param jobs_dir: path of the directory with job definitions :param whitelists_dir: path of the directory with whitelists definitions (aka test-plans) :param data_dir: path of the directory with files used by jobs at runtime :param bin_dir: path of the directory with additional executables :param locale_dir: path of the directory with locale database (translation catalogs) :param base_dir: path of the directory with (perhaps) all of jobs_dir, whitelist_dir, data_dir, bin_dir, locale_dir. This may be None. This is also the effective value of $CHECKBOX_SHARE """ # Meta-data self._name = name self._version = version self._description = description self._secure = secure self._gettext_domain = gettext_domain # Directories self._jobs_dir = jobs_dir self._whitelists_dir = whitelists_dir self._data_dir = data_dir self._bin_dir = bin_dir self._locale_dir = locale_dir self._base_dir = base_dir # Loaded data if self.whitelists_dir is not None: whitelists_dir_list = [self.whitelists_dir] else: whitelists_dir_list = [] self._whitelist_collection = FsPlugInCollection( whitelists_dir_list, ext=".whitelist", wrapper=WhiteListPlugIn, implicit_namespace=self.namespace) if self.jobs_dir is not None: jobs_dir_list = [self.jobs_dir] else: jobs_dir_list = [] self._job_collection = FsPlugInCollection( jobs_dir_list, ext=(".txt", ".txt.in"), wrapper=JobDefinitionPlugIn, provider=self) # Setup translations if gettext_domain and locale_dir: gettext.bindtextdomain(self._gettext_domain, self._locale_dir)
class Provider1(IProvider1, IProviderBackend1): """ A v1 provider implementation. A provider is a container of jobs and whitelists. It provides additional meta-data and knows about location of essential directories to both load structured data and provide runtime information for job execution. Providers are normally loaded with :class:`Provider1PlugIn`, due to the number of fields involved in basic initialization. """ def __init__(self, name, version, description, secure, gettext_domain, jobs_dir, whitelists_dir, data_dir, bin_dir, locale_dir, base_dir): """ Initialize a provider with a set of meta-data and directories. :param name: provider name / ID :param version: provider version :param description: provider version This is the untranslated version of this field. Implementations may obtain the localized version based on the gettext_domain property. :param secure: secure bit When True jobs from this provider should be available via the trusted launcher mechanism. It should be set to True for system-wide installed providers. :param gettext_domain: gettext domain that contains translations for this provider :param jobs_dir: path of the directory with job definitions :param whitelists_dir: path of the directory with whitelists definitions (aka test-plans) :param data_dir: path of the directory with files used by jobs at runtime :param bin_dir: path of the directory with additional executables :param locale_dir: path of the directory with locale database (translation catalogs) :param base_dir: path of the directory with (perhaps) all of jobs_dir, whitelist_dir, data_dir, bin_dir, locale_dir. This may be None. This is also the effective value of $CHECKBOX_SHARE """ # Meta-data self._name = name self._version = version self._description = description self._secure = secure self._gettext_domain = gettext_domain # Directories self._jobs_dir = jobs_dir self._whitelists_dir = whitelists_dir self._data_dir = data_dir self._bin_dir = bin_dir self._locale_dir = locale_dir self._base_dir = base_dir # Loaded data if self.whitelists_dir is not None: whitelists_dir_list = [self.whitelists_dir] else: whitelists_dir_list = [] self._whitelist_collection = FsPlugInCollection( whitelists_dir_list, ext=".whitelist", wrapper=WhiteListPlugIn, implicit_namespace=self.namespace) if self.jobs_dir is not None: jobs_dir_list = [self.jobs_dir] else: jobs_dir_list = [] self._job_collection = FsPlugInCollection( jobs_dir_list, ext=(".txt", ".txt.in"), wrapper=JobDefinitionPlugIn, provider=self) # Setup translations if gettext_domain and locale_dir: gettext.bindtextdomain(self._gettext_domain, self._locale_dir) @classmethod def from_definition(cls, definition, secure): """ Initialize a provider from Provider1Definition object :param definition: A Provider1Definition object to use as reference :param secure: Value of the secure flag. This cannot be expressed by a definition object. This method simplifies initialization of a Provider1 object where the caller already has a Provider1Definition object. Depending on the value of ``definition.location`` all of the directories are either None or initialized to a *good* (typical) value relative to *location* The only value that you may want to adjust, for working with source providers, is *locale_dir*, by default it would be ``location/locale`` but ``manage.py i18n`` creates ``location/build/mo`` """ logger.debug("Loading provider from definition %r", definition) # Initialize the provider object return cls( definition.name, definition.version, definition.description, secure, definition.effective_gettext_domain, definition.effective_jobs_dir, definition.effective_whitelists_dir, definition.effective_data_dir, definition.effective_bin_dir, definition.effective_locale_dir, definition.location or None) def __repr__(self): return "<{} name:{!r}>".format(self.__class__.__name__, self.name) @property def name(self): """ name of this provider """ return self._name @property def namespace(self): """ namespace component of the provider name This property defines the namespace in which all provider jobs are defined in. Jobs within one namespace do not need to be fully qualified by prefixing their partial identifier with provider namespace (so all stays 'as-is'). Jobs that need to interact with other provider namespaces need to use the fully qualified job identifier instead. The identifier is defined as the part of the provider name, up to the colon. This effectively gives organizations flat namespace within one year-domain pair and allows to create private namespaces by using sub-domains. """ return self._name.split(':', 1)[0] @property def version(self): """ version of this provider """ return self._version @property def description(self): """ description of this provider """ return self._description def tr_description(self): """ Get the translated version of :meth:`description` """ return self.get_translated_data(self.description) @property def jobs_dir(self): """ absolute path of the jobs directory """ return self._jobs_dir @property def whitelists_dir(self): """ absolute path of the whitelist directory """ return self._whitelists_dir @property def data_dir(self): """ absolute path of the data directory """ return self._data_dir @property def bin_dir(self): """ absolute path of the bin directory .. note:: The programs in that directory may not work without setting PYTHONPATH and CHECKBOX_SHARE. """ return self._bin_dir @property def locale_dir(self): """ absolute path of the directory with locale data The value is applicable as argument bindtextdomain() """ return self._locale_dir @property def base_dir(self): """ path of the directory with (perhaps) all of jobs_dir, whitelist_dir, data_dir, bin_dir, locale_dir. This may be None """ return self._base_dir @property def CHECKBOX_SHARE(self): """ required value of CHECKBOX_SHARE environment variable. .. note:: This variable is only required by one script. It would be nice to remove this later on. """ return self.base_dir @property def extra_PYTHONPATH(self): """ additional entry for PYTHONPATH, if needed. This entry is required for CheckBox scripts to import the correct CheckBox python libraries. .. note:: The result may be None """ return None @property def secure(self): """ flag indicating that this provider was loaded from the secure portion of PROVIDERPATH and thus can be used with the plainbox-trusted-launcher-1. """ return self._secure @property def gettext_domain(self): """ the name of the gettext domain associated with this provider This value may be empty, in such case provider data cannot be localized for the user environment. """ return self._gettext_domain def get_builtin_whitelists(self): """ Load all the whitelists from :attr:`whitelists_dir` and return them This method looks at the whitelist directory and loads all files ending with .whitelist as a WhiteList object. :returns: A list of :class:`~plainbox.impl.secure.qualifiers.WhiteList` objects sorted by :attr:`plainbox.impl.secure.qualifiers.WhiteList.name`. :raises IOError, OSError: if there were any problems accessing files or directories. Note that OSError is silently ignored when the `whitelists_dir` directory is missing. """ self._whitelist_collection.load() if self._whitelist_collection.problem_list: raise self._whitelist_collection.problem_list[0] else: return sorted(self._whitelist_collection.get_all_plugin_objects(), key=lambda whitelist: whitelist.name) def get_builtin_jobs(self): """ Load and parse all of the job definitions of this provider. :returns: A sorted list of JobDefinition objects :raises RFC822SyntaxError: if any of the loaded files was not valid RFC822 :raises IOError, OSError: if there were any problems accessing files or directories. Note that OSError is silently ignored when the `jobs_dir` directory is missing. ..note:: This method should not be used anymore. Consider transitioning your code to :meth:`load_all_jobs()` which is more reliable. """ job_list, problem_list = self.load_all_jobs() if problem_list: raise problem_list[0] else: return job_list def load_all_jobs(self): """ Load and parse all of the job definitions of this provider. Unlike :meth:`get_builtin_jobs()` this method does not stop after the first problem encountered and instead collects all of the problems into a list which is returned alongside the job list. :returns: Pair (job_list, problem_list) where each job_list is a sorted list of JobDefinition objects and each item from problem_list is an exception. """ self._job_collection.load() job_list = sorted( itertools.chain( *self._job_collection.get_all_plugin_objects()), key=lambda job: job.id) problem_list = self._job_collection.problem_list return job_list, problem_list def get_all_executables(self): """ Discover and return all executables offered by this provider :returns: list of executable names (without the full path) :raises IOError, OSError: if there were any problems accessing files or directories. Note that OSError is silently ignored when the `bin_dir` directory is missing. """ executable_list = [] if self.bin_dir is None: return executable_list try: items = os.listdir(self.bin_dir) except OSError as exc: if exc.errno == errno.ENOENT: items = [] else: raise for name in items: filename = os.path.join(self.bin_dir, name) if os.access(filename, os.F_OK | os.X_OK): executable_list.append(filename) return sorted(executable_list) def get_translated_data(self, msgid): """ Get a localized piece of data :param msgid: data to translate :returns: translated data obtained from the provider if msgid is not False (empty string and None both are) and this provider has a gettext_domain defined for it, msgid itself otherwise. """ if msgid and self._gettext_domain: return gettext.dgettext(self._gettext_domain, msgid) else: return msgid
class Provider1(IProvider1, IProviderBackend1): """ A v1 provider implementation. This base class implements a checkbox-like provider object. Subclasses are only required to implement a single method that designates the base location for all other data. """ def __init__(self, base_dir, name, version, description, secure, gettext_domain=None): """ Initialize the provider with the associated base directory. All of the typical v1 provider data is relative to this directory. It can be customized by subclassing and overriding the particular methods of the IProviderBackend1 class but that should not be necessary in normal operation. """ self._base_dir = base_dir self._name = name self._version = version self._description = description self._secure = secure self._gettext_domain = gettext_domain self._whitelist_collection = FsPlugInCollection( [self.whitelists_dir], ext=".whitelist", wrapper=WhiteListPlugIn) self._job_collection = FsPlugInCollection( [self.jobs_dir], ext=(".txt", ".txt.in"), wrapper=JobDefinitionPlugIn, provider=self) def __repr__(self): return "<{} name:{!r} base_dir:{!r}>".format( self.__class__.__name__, self.name, self.base_dir) @property def base_dir(self): """ pathname to a directory with essential provider data This pathname is used for deriving :attr:`jobs_dir`, :attr:`bin_dir` and :attr:`whitelists_dir`. """ return self._base_dir @property def name(self): """ name of this provider """ return self._name @property def version(self): """ version of this provider """ return self._version @property def description(self): """ description of this provider """ return self._description @property def jobs_dir(self): """ Return an absolute path of the jobs directory """ return os.path.join(self._base_dir, "jobs") @property def bin_dir(self): """ Return an absolute path of the bin directory .. note:: The programs in that directory may not work without setting PYTHONPATH and CHECKBOX_SHARE. """ return os.path.join(self._base_dir, "bin") @property def whitelists_dir(self): """ Return an absolute path of the whitelist directory """ return os.path.join(self._base_dir, "whitelists") @property def CHECKBOX_SHARE(self): """ Return the required value of CHECKBOX_SHARE environment variable. .. note:: This variable is only required by one script. It would be nice to remove this later on. """ return self._base_dir @property def extra_PYTHONPATH(self): """ Return additional entry for PYTHONPATH, if needed. This entry is required for CheckBox scripts to import the correct CheckBox python libraries. .. note:: The result may be None """ return None @property def secure(self): """ flag indicating that this provider was loaded from the secure portion of PROVIDERPATH and thus can be used with the plainbox-trusted-launcher-1. """ return self._secure @property def gettext_domain(self): """ the name of the gettext domain associated with this provider This value may be empty, in such case provider data cannot be localized for the user environment. """ return self._gettext_domain def get_builtin_whitelists(self): """ Load all the whitelists from :attr:`whitelists_dir` and return them This method looks at the whitelist directory and loads all files ending with .whitelist as a WhiteList object. :returns: A list of :class:`~plainbox.impl.secure.qualifiers.WhiteList` objects sorted by :attr:`plainbox.impl.secure.qualifiers.WhiteList.name`. :raises IOError, OSError: if there were any problems accessing files or directories. Note that OSError is silently ignored when the `whitelists_dir` directory is missing. """ self._whitelist_collection.load() if self._whitelist_collection.problem_list: raise self._whitelist_collection.problem_list[0] else: return sorted(self._whitelist_collection.get_all_plugin_objects(), key=lambda whitelist: whitelist.name) def get_builtin_jobs(self): """ Load and parse all of the job definitions of this provider. :returns: A sorted list of JobDefinition objects :raises RFC822SyntaxError: if any of the loaded files was not valid RFC822 :raises IOError, OSError: if there were any problems accessing files or directories. Note that OSError is silently ignored when the `jobs_dir` directory is missing. ..note:: This method should not be used anymore. Consider transitioning your code to :meth:`load_all_jobs()` which is more reliable. """ job_list, problem_list = self.load_all_jobs() if problem_list: raise problem_list[0] else: return job_list def load_all_jobs(self): """ Load and parse all of the job definitions of this provider. Unlike :meth:`get_builtin_jobs()` this method does not stop after the first problem encountered and instead collects all of the problems into a list which is returned alongside the job list. :returns: Pair (job_list, problem_list) where each job_list is a sorted list of JobDefinition objects and each item from problem_list is an exception. """ self._job_collection.load() job_list = sorted( itertools.chain( *self._job_collection.get_all_plugin_objects()), key=lambda job: job.name) problem_list = self._job_collection.problem_list return job_list, problem_list def get_all_executables(self): """ Discover and return all executables offered by this provider :returns: list of executable names (without the full path) :raises IOError, OSError: if there were any problems accessing files or directories. Note that OSError is silently ignored when the `bin_dir` directory is missing. """ executable_list = [] try: items = os.listdir(self.bin_dir) except OSError as exc: if exc.errno == errno.ENOENT: items = [] else: raise for name in items: filename = os.path.join(self.bin_dir, name) if os.access(filename, os.F_OK | os.X_OK): executable_list.append(filename) return sorted(executable_list)
class FsPlugInCollectionTests(TestCase): _P1 = "/system/providers" _P2 = "home/user/.providers" _DIR_LIST = [_P1, _P2] _EXT = ".plugin" def setUp(self): # Create a collection self.col = FsPlugInCollection(self._DIR_LIST, self._EXT) def test_path_is_set(self): # Ensure that path was saved self.assertEqual(self.col._dir_list, self._DIR_LIST) def test_ext_is_set(self): # Ensure that ext was saved self.assertEqual(self.col._ext, self._EXT) def test_plugins_are_empty(self): # Ensure that plugins start out empty self.assertEqual(len(self.col._plugins), 0) def test_initial_loaded_flag(self): # Ensure that 'loaded' flag is false self.assertFalse(self.col._loaded) def test_default_wrapper(self): # Ensure that the wrapper is :class:`PlugIn` self.assertEqual(self.col._wrapper, PlugIn) @mock.patch('plainbox.impl.secure.plugins.logger') @mock.patch('builtins.open') @mock.patch('os.path.isfile') @mock.patch('os.listdir') def test_load(self, mock_listdir, mock_isfile, mock_open, mock_logger): # Mock a bit of filesystem access methods to make some plugins show up def fake_listdir(path): if path == self._P1: return [ # A regular plugin 'foo.plugin', # Another regular plugin 'bar.plugin', # Unrelated file, not a plugin 'unrelated.txt', # A directory that looks like a plugin 'dir.bad.plugin', # A plugin without read permissions 'noperm.plugin'] else: raise OSError("There is nothing in {}".format(path)) def fake_isfile(path): return not os.path.basename(path).startswith('dir.') def fake_open(path, encoding=None, mode=None): m = mock.MagicMock(name='opened file {!r}'.format(path)) m.__enter__.return_value = m if path == os.path.join(self._P1, 'foo.plugin'): m.read.return_value = "foo" return m elif path == os.path.join(self._P1, 'bar.plugin'): m.read.return_value = "bar" return m elif path == os.path.join(self._P1, 'noperm.plugin'): raise OSError("You cannot open this file") else: raise IOError("Unexpected file: {}".format(path)) mock_listdir.side_effect = fake_listdir mock_isfile.side_effect = fake_isfile mock_open.side_effect = fake_open # Load all plugins now self.col.load() # And 'again', just to ensure we're doing the IO only once self.col.load() # Ensure that we actually tried to look at the filesytstem self.assertEqual( mock_listdir.call_args_list, [ ((self._P1, ), {}), ((self._P2, ), {}) ]) # Ensure that we actually tried to check if things are files self.assertEqual( mock_isfile.call_args_list, [ ((os.path.join(self._P1, 'foo.plugin'),), {}), ((os.path.join(self._P1, 'bar.plugin'),), {}), ((os.path.join(self._P1, 'dir.bad.plugin'),), {}), ((os.path.join(self._P1, 'noperm.plugin'),), {}), ]) # Ensure that we actually tried to open some files self.assertEqual( mock_open.call_args_list, [ ((os.path.join(self._P1, 'bar.plugin'),), {'encoding': 'UTF-8'}), ((os.path.join(self._P1, 'foo.plugin'),), {'encoding': 'UTF-8'}), ((os.path.join(self._P1, 'noperm.plugin'),), {'encoding': 'UTF-8'}), ]) # Ensure that an exception was logged mock_logger.error.assert_called_with( 'Unable to load %r: %s', '/system/providers/noperm.plugin', 'You cannot open this file') # Ensure that all of the errors are collected # Using repr() since OSError seems hard to compare correctly self.assertEqual( repr(self.col.problem_list[0]), repr(OSError('You cannot open this file'))) @mock.patch('plainbox.impl.secure.plugins.logger') @mock.patch('builtins.open') @mock.patch('os.path.isfile') @mock.patch('os.listdir') def test_load__two_extensions(self, mock_listdir, mock_isfile, mock_open, mock_logger): """ verify that FsPlugInCollection works with multiple extensions """ mock_listdir.return_value = ["foo.txt", "bar.txt.in"] mock_isfile.return_value = True def fake_open(path, encoding=None, mode=None): m = mock.MagicMock(name='opened file {!r}'.format(path)) m.read.return_value = "text" m.__enter__.return_value = m return m mock_open.side_effect = fake_open # Create a collection that looks for both extensions col = FsPlugInCollection([self._P1], (".txt", ".txt.in")) # Load everything col.load() # Ensure that we actually tried to look at the filesystem self.assertEqual( mock_listdir.call_args_list, [ ((self._P1, ), {}), ]) # Ensure that we actually tried to check if things are files self.assertEqual( mock_isfile.call_args_list, [ ((os.path.join(self._P1, 'foo.txt'),), {}), ((os.path.join(self._P1, 'bar.txt.in'),), {}), ]) # Ensure that we actually tried to open some files self.assertEqual( mock_open.call_args_list, [ ((os.path.join(self._P1, 'bar.txt.in'),), {'encoding': 'UTF-8'}), ((os.path.join(self._P1, 'foo.txt'),), {'encoding': 'UTF-8'}), ]) # Ensure that no exception was logged mock_logger.error.assert_not_called() # Ensure that everything was okay self.assertEqual(col.problem_list, []) # Ensure that both files got added self.assertEqual( col.get_by_name( os.path.join(self._P1, "foo.txt") ).plugin_object, "text") self.assertEqual( col.get_by_name( os.path.join(self._P1, "bar.txt.in") ).plugin_object, "text")
def __init__(self, name, version, description, secure, gettext_domain, jobs_dir, whitelists_dir, data_dir, bin_dir, locale_dir, base_dir): """ Initialize a provider with a set of meta-data and directories. :param name: provider name / ID :param version: provider version :param description: provider version This is the untranslated version of this field. Implementations may obtain the localized version based on the gettext_domain property. :param secure: secure bit When True jobs from this provider should be available via the trusted launcher mechanism. It should be set to True for system-wide installed providers. :param gettext_domain: gettext domain that contains translations for this provider :param jobs_dir: path of the directory with job definitions :param whitelists_dir: path of the directory with whitelists definitions (aka test-plans) :param data_dir: path of the directory with files used by jobs at runtime :param bin_dir: path of the directory with additional executables :param locale_dir: path of the directory with locale database (translation catalogs) :param base_dir: path of the directory with (perhaps) all of jobs_dir, whitelist_dir, data_dir, bin_dir, locale_dir. This may be None. This is also the effective value of $CHECKBOX_SHARE """ # Meta-data self._name = name self._version = version self._description = description self._secure = secure self._gettext_domain = gettext_domain # Directories self._jobs_dir = jobs_dir self._whitelists_dir = whitelists_dir self._data_dir = data_dir self._bin_dir = bin_dir self._locale_dir = locale_dir self._base_dir = base_dir # Loaded data if self.whitelists_dir is not None: whitelists_dir_list = [self.whitelists_dir] else: whitelists_dir_list = [] self._whitelist_collection = FsPlugInCollection( whitelists_dir_list, ext=".whitelist", wrapper=WhiteListPlugIn, implicit_namespace=self.namespace) if self.jobs_dir is not None: jobs_dir_list = [self.jobs_dir] else: jobs_dir_list = [] self._job_collection = FsPlugInCollection(jobs_dir_list, ext=(".txt", ".txt.in"), wrapper=JobDefinitionPlugIn, provider=self) # Setup translations if gettext_domain and locale_dir: gettext.bindtextdomain(self._gettext_domain, self._locale_dir)
class Provider1(IProvider1, IProviderBackend1): """ A v1 provider implementation. A provider is a container of jobs and whitelists. It provides additional meta-data and knows about location of essential directories to both load structured data and provide runtime information for job execution. Providers are normally loaded with :class:`Provider1PlugIn`, due to the number of fields involved in basic initialization. """ def __init__(self, name, version, description, secure, gettext_domain, jobs_dir, whitelists_dir, data_dir, bin_dir, locale_dir, base_dir): """ Initialize a provider with a set of meta-data and directories. :param name: provider name / ID :param version: provider version :param description: provider version This is the untranslated version of this field. Implementations may obtain the localized version based on the gettext_domain property. :param secure: secure bit When True jobs from this provider should be available via the trusted launcher mechanism. It should be set to True for system-wide installed providers. :param gettext_domain: gettext domain that contains translations for this provider :param jobs_dir: path of the directory with job definitions :param whitelists_dir: path of the directory with whitelists definitions (aka test-plans) :param data_dir: path of the directory with files used by jobs at runtime :param bin_dir: path of the directory with additional executables :param locale_dir: path of the directory with locale database (translation catalogs) :param base_dir: path of the directory with (perhaps) all of jobs_dir, whitelist_dir, data_dir, bin_dir, locale_dir. This may be None. This is also the effective value of $CHECKBOX_SHARE """ # Meta-data self._name = name self._version = version self._description = description self._secure = secure self._gettext_domain = gettext_domain # Directories self._jobs_dir = jobs_dir self._whitelists_dir = whitelists_dir self._data_dir = data_dir self._bin_dir = bin_dir self._locale_dir = locale_dir self._base_dir = base_dir # Loaded data if self.whitelists_dir is not None: whitelists_dir_list = [self.whitelists_dir] else: whitelists_dir_list = [] self._whitelist_collection = FsPlugInCollection( whitelists_dir_list, ext=".whitelist", wrapper=WhiteListPlugIn, implicit_namespace=self.namespace) if self.jobs_dir is not None: jobs_dir_list = [self.jobs_dir] else: jobs_dir_list = [] self._job_collection = FsPlugInCollection(jobs_dir_list, ext=(".txt", ".txt.in"), wrapper=JobDefinitionPlugIn, provider=self) # Setup translations if gettext_domain and locale_dir: gettext.bindtextdomain(self._gettext_domain, self._locale_dir) @classmethod def from_definition(cls, definition, secure): """ Initialize a provider from Provider1Definition object :param definition: A Provider1Definition object to use as reference :param secure: Value of the secure flag. This cannot be expressed by a definition object. This method simplifies initialization of a Provider1 object where the caller already has a Provider1Definition object. Depending on the value of ``definition.location`` all of the directories are either None or initialized to a *good* (typical) value relative to *location* The only value that you may want to adjust, for working with source providers, is *locale_dir*, by default it would be ``location/locale`` but ``manage.py i18n`` creates ``location/build/mo`` """ logger.debug("Loading provider from definition %r", definition) # Initialize the provider object return cls(definition.name, definition.version, definition.description, secure, definition.effective_gettext_domain, definition.effective_jobs_dir, definition.effective_whitelists_dir, definition.effective_data_dir, definition.effective_bin_dir, definition.effective_locale_dir, definition.location or None) def __repr__(self): return "<{} name:{!r}>".format(self.__class__.__name__, self.name) @property def name(self): """ name of this provider """ return self._name @property def namespace(self): """ namespace component of the provider name This property defines the namespace in which all provider jobs are defined in. Jobs within one namespace do not need to be fully qualified by prefixing their partial identifier with provider namespace (so all stays 'as-is'). Jobs that need to interact with other provider namespaces need to use the fully qualified job identifier instead. The identifier is defined as the part of the provider name, up to the colon. This effectively gives organizations flat namespace within one year-domain pair and allows to create private namespaces by using sub-domains. """ return self._name.split(':', 1)[0] @property def version(self): """ version of this provider """ return self._version @property def description(self): """ description of this provider """ return self._description def tr_description(self): """ Get the translated version of :meth:`description` """ return self.get_translated_data(self.description) @property def jobs_dir(self): """ absolute path of the jobs directory """ return self._jobs_dir @property def whitelists_dir(self): """ absolute path of the whitelist directory """ return self._whitelists_dir @property def data_dir(self): """ absolute path of the data directory """ return self._data_dir @property def bin_dir(self): """ absolute path of the bin directory .. note:: The programs in that directory may not work without setting PYTHONPATH and CHECKBOX_SHARE. """ return self._bin_dir @property def locale_dir(self): """ absolute path of the directory with locale data The value is applicable as argument bindtextdomain() """ return self._locale_dir @property def base_dir(self): """ path of the directory with (perhaps) all of jobs_dir, whitelist_dir, data_dir, bin_dir, locale_dir. This may be None """ return self._base_dir @property def CHECKBOX_SHARE(self): """ required value of CHECKBOX_SHARE environment variable. .. note:: This variable is only required by one script. It would be nice to remove this later on. """ return self.base_dir @property def extra_PYTHONPATH(self): """ additional entry for PYTHONPATH, if needed. This entry is required for CheckBox scripts to import the correct CheckBox python libraries. .. note:: The result may be None """ return None @property def secure(self): """ flag indicating that this provider was loaded from the secure portion of PROVIDERPATH and thus can be used with the plainbox-trusted-launcher-1. """ return self._secure @property def gettext_domain(self): """ the name of the gettext domain associated with this provider This value may be empty, in such case provider data cannot be localized for the user environment. """ return self._gettext_domain def get_builtin_whitelists(self): """ Load all the whitelists from :attr:`whitelists_dir` and return them This method looks at the whitelist directory and loads all files ending with .whitelist as a WhiteList object. :returns: A list of :class:`~plainbox.impl.secure.qualifiers.WhiteList` objects sorted by :attr:`plainbox.impl.secure.qualifiers.WhiteList.name`. :raises IOError, OSError: if there were any problems accessing files or directories. Note that OSError is silently ignored when the `whitelists_dir` directory is missing. """ self._whitelist_collection.load() if self._whitelist_collection.problem_list: raise self._whitelist_collection.problem_list[0] else: return sorted(self._whitelist_collection.get_all_plugin_objects(), key=lambda whitelist: whitelist.name) def get_builtin_jobs(self): """ Load and parse all of the job definitions of this provider. :returns: A sorted list of JobDefinition objects :raises RFC822SyntaxError: if any of the loaded files was not valid RFC822 :raises IOError, OSError: if there were any problems accessing files or directories. Note that OSError is silently ignored when the `jobs_dir` directory is missing. ..note:: This method should not be used anymore. Consider transitioning your code to :meth:`load_all_jobs()` which is more reliable. """ job_list, problem_list = self.load_all_jobs() if problem_list: raise problem_list[0] else: return job_list def load_all_jobs(self): """ Load and parse all of the job definitions of this provider. Unlike :meth:`get_builtin_jobs()` this method does not stop after the first problem encountered and instead collects all of the problems into a list which is returned alongside the job list. :returns: Pair (job_list, problem_list) where each job_list is a sorted list of JobDefinition objects and each item from problem_list is an exception. """ self._job_collection.load() job_list = sorted( itertools.chain(*self._job_collection.get_all_plugin_objects()), key=lambda job: job.id) problem_list = self._job_collection.problem_list return job_list, problem_list def get_all_executables(self): """ Discover and return all executables offered by this provider :returns: list of executable names (without the full path) :raises IOError, OSError: if there were any problems accessing files or directories. Note that OSError is silently ignored when the `bin_dir` directory is missing. """ executable_list = [] if self.bin_dir is None: return executable_list try: items = os.listdir(self.bin_dir) except OSError as exc: if exc.errno == errno.ENOENT: items = [] else: raise for name in items: filename = os.path.join(self.bin_dir, name) if os.access(filename, os.F_OK | os.X_OK): executable_list.append(filename) return sorted(executable_list) def get_translated_data(self, msgid): """ Get a localized piece of data :param msgid: data to translate :returns: translated data obtained from the provider if msgid is not False (empty string and None both are) and this provider has a gettext_domain defined for it, msgid itself otherwise. """ if msgid and self._gettext_domain: return gettext.dgettext(self._gettext_domain, msgid) else: return msgid