def test_load__two_extensions(self, mock_listdir, mock_isfile, mock_open,
mock_logger):
"""
verify that FsPlugInCollection works with multiple extensions
"""
mock_listdir.return_value = ["foo.txt", "bar.txt.in"]
mock_isfile.return_value = True
def fake_open(path, encoding=None, mode=None):
m = mock.MagicMock(name='opened file {!r}'.format(path))
m.read.return_value = "text"
m.__enter__.return_value = m
return m
mock_open.side_effect = fake_open
# Create a collection that looks for both extensions
col = FsPlugInCollection([self._P1], (".txt", ".txt.in"))
# Load everything
col.load()
# Ensure that we actually tried to look at the filesystem
self.assertEqual(
mock_listdir.call_args_list, [
((self._P1, ), {}),
])
# Ensure that we actually tried to check if things are files
self.assertEqual(
mock_isfile.call_args_list, [
((os.path.join(self._P1, 'foo.txt'),), {}),
((os.path.join(self._P1, 'bar.txt.in'),), {}),
])
# Ensure that we actually tried to open some files
self.assertEqual(
mock_open.call_args_list, [
((os.path.join(self._P1, 'bar.txt.in'),),
{'encoding': 'UTF-8'}),
((os.path.join(self._P1, 'foo.txt'),),
{'encoding': 'UTF-8'}),
])
# Ensure that no exception was logged
mock_logger.error.assert_not_called()
# Ensure that everything was okay
self.assertEqual(col.problem_list, [])
# Ensure that both files got added
self.assertEqual(
col.get_by_name(
os.path.join(self._P1, "foo.txt")
).plugin_object, "text")
self.assertEqual(
col.get_by_name(
os.path.join(self._P1, "bar.txt.in")
).plugin_object, "text")
def __init__(self, base_dir, name, version, description, secure,
gettext_domain=None):
"""
Initialize the provider with the associated base directory.
All of the typical v1 provider data is relative to this directory. It
can be customized by subclassing and overriding the particular methods
of the IProviderBackend1 class but that should not be necessary in
normal operation.
"""
self._base_dir = base_dir
self._name = name
self._version = version
self._description = description
self._secure = secure
self._gettext_domain = gettext_domain
self._whitelist_collection = FsPlugInCollection(
[self.whitelists_dir], ext=".whitelist", wrapper=WhiteListPlugIn)
self._job_collection = FsPlugInCollection(
[self.jobs_dir], ext=(".txt", ".txt.in"),
wrapper=JobDefinitionPlugIn, provider=self)
def test_load__two_extensions(self, mock_listdir, mock_isfile, mock_open,
mock_logger):
"""
verify that FsPlugInCollection works with multiple extensions
"""
mock_listdir.return_value = ["foo.txt", "bar.txt.in"]
mock_isfile.return_value = True
def fake_open(path, encoding=None, mode=None):
m = mock.MagicMock(name='opened file {!r}'.format(path))
m.read.return_value = "text"
m.__enter__.return_value = m
return m
mock_open.side_effect = fake_open
# Create a collection that looks for both extensions
col = FsPlugInCollection([self._P1], (".txt", ".txt.in"))
# Load everything
col.load()
# Ensure that we actually tried to look at the filesystem
self.assertEqual(mock_listdir.call_args_list, [
((self._P1, ), {}),
])
# Ensure that we actually tried to check if things are files
self.assertEqual(mock_isfile.call_args_list, [
((os.path.join(self._P1, 'foo.txt'), ), {}),
((os.path.join(self._P1, 'bar.txt.in'), ), {}),
])
# Ensure that we actually tried to open some files
self.assertEqual(mock_open.call_args_list, [
((os.path.join(self._P1, 'bar.txt.in'), ), {
'encoding': 'UTF-8'
}),
((os.path.join(self._P1, 'foo.txt'), ), {
'encoding': 'UTF-8'
}),
])
# Ensure that no exception was logged
self.assertEqual(mock_logger.error.mock_calls, [])
# Ensure that everything was okay
self.assertEqual(col.problem_list, [])
# Ensure that both files got added
self.assertEqual(
col.get_by_name(os.path.join(self._P1, "foo.txt")).plugin_object,
"text")
self.assertEqual(
col.get_by_name(os.path.join(self._P1,
"bar.txt.in")).plugin_object, "text")
def setUp(self):
# Create a collection
self.col = FsPlugInCollection(self._DIR_LIST, self._EXT)
class FsPlugInCollectionTests(TestCase):
_P1 = "/system/providers"
_P2 = "home/user/.providers"
_DIR_LIST = [_P1, _P2]
_EXT = ".plugin"
def setUp(self):
# Create a collection
self.col = FsPlugInCollection(self._DIR_LIST, self._EXT)
def test_path_is_set(self):
# Ensure that path was saved
self.assertEqual(self.col._dir_list, self._DIR_LIST)
def test_ext_is_set(self):
# Ensure that ext was saved
self.assertEqual(self.col._ext, self._EXT)
def test_plugins_are_empty(self):
# Ensure that plugins start out empty
self.assertEqual(len(self.col._plugins), 0)
def test_initial_loaded_flag(self):
# Ensure that 'loaded' flag is false
self.assertFalse(self.col._loaded)
def test_default_wrapper(self):
# Ensure that the wrapper is :class:`PlugIn`
self.assertEqual(self.col._wrapper, PlugIn)
@mock.patch('plainbox.impl.secure.plugins.logger')
@mock.patch('builtins.open')
@mock.patch('os.path.isfile')
@mock.patch('os.listdir')
def test_load(self, mock_listdir, mock_isfile, mock_open, mock_logger):
# Mock a bit of filesystem access methods to make some plugins show up
def fake_listdir(path):
if path == self._P1:
return [
# A regular plugin
'foo.plugin',
# Another regular plugin
'bar.plugin',
# Unrelated file, not a plugin
'unrelated.txt',
# A directory that looks like a plugin
'dir.bad.plugin',
# A plugin without read permissions
'noperm.plugin'
]
else:
raise OSError("There is nothing in {}".format(path))
def fake_isfile(path):
return not os.path.basename(path).startswith('dir.')
def fake_open(path, encoding=None, mode=None):
m = mock.MagicMock(name='opened file {!r}'.format(path))
m.__enter__.return_value = m
if path == os.path.join(self._P1, 'foo.plugin'):
m.read.return_value = "foo"
return m
elif path == os.path.join(self._P1, 'bar.plugin'):
m.read.return_value = "bar"
return m
elif path == os.path.join(self._P1, 'noperm.plugin'):
raise OSError("You cannot open this file")
else:
raise IOError("Unexpected file: {}".format(path))
mock_listdir.side_effect = fake_listdir
mock_isfile.side_effect = fake_isfile
mock_open.side_effect = fake_open
# Load all plugins now
self.col.load()
# And 'again', just to ensure we're doing the IO only once
self.col.load()
# Ensure that we actually tried to look at the filesytstem
self.assertEqual(mock_listdir.call_args_list, [((self._P1, ), {}),
((self._P2, ), {})])
# Ensure that we actually tried to check if things are files
self.assertEqual(mock_isfile.call_args_list, [
((os.path.join(self._P1, 'foo.plugin'), ), {}),
((os.path.join(self._P1, 'bar.plugin'), ), {}),
((os.path.join(self._P1, 'dir.bad.plugin'), ), {}),
((os.path.join(self._P1, 'noperm.plugin'), ), {}),
])
# Ensure that we actually tried to open some files
self.assertEqual(mock_open.call_args_list, [
((os.path.join(self._P1, 'bar.plugin'), ), {
'encoding': 'UTF-8'
}),
((os.path.join(self._P1, 'foo.plugin'), ), {
'encoding': 'UTF-8'
}),
((os.path.join(self._P1, 'noperm.plugin'), ), {
'encoding': 'UTF-8'
}),
])
# Ensure that an exception was logged
mock_logger.error.assert_called_with(
'Unable to load %r: %s', '/system/providers/noperm.plugin',
'You cannot open this file')
# Ensure that all of the errors are collected
# Using repr() since OSError seems hard to compare correctly
self.assertEqual(repr(self.col.problem_list[0]),
repr(OSError('You cannot open this file')))
@mock.patch('plainbox.impl.secure.plugins.logger')
@mock.patch('builtins.open')
@mock.patch('os.path.isfile')
@mock.patch('os.listdir')
def test_load__two_extensions(self, mock_listdir, mock_isfile, mock_open,
mock_logger):
"""
verify that FsPlugInCollection works with multiple extensions
"""
mock_listdir.return_value = ["foo.txt", "bar.txt.in"]
mock_isfile.return_value = True
def fake_open(path, encoding=None, mode=None):
m = mock.MagicMock(name='opened file {!r}'.format(path))
m.read.return_value = "text"
m.__enter__.return_value = m
return m
mock_open.side_effect = fake_open
# Create a collection that looks for both extensions
col = FsPlugInCollection([self._P1], (".txt", ".txt.in"))
# Load everything
col.load()
# Ensure that we actually tried to look at the filesystem
self.assertEqual(mock_listdir.call_args_list, [
((self._P1, ), {}),
])
# Ensure that we actually tried to check if things are files
self.assertEqual(mock_isfile.call_args_list, [
((os.path.join(self._P1, 'foo.txt'), ), {}),
((os.path.join(self._P1, 'bar.txt.in'), ), {}),
])
# Ensure that we actually tried to open some files
self.assertEqual(mock_open.call_args_list, [
((os.path.join(self._P1, 'bar.txt.in'), ), {
'encoding': 'UTF-8'
}),
((os.path.join(self._P1, 'foo.txt'), ), {
'encoding': 'UTF-8'
}),
])
# Ensure that no exception was logged
self.assertEqual(mock_logger.error.mock_calls, [])
# Ensure that everything was okay
self.assertEqual(col.problem_list, [])
# Ensure that both files got added
self.assertEqual(
col.get_by_name(os.path.join(self._P1, "foo.txt")).plugin_object,
"text")
self.assertEqual(
col.get_by_name(os.path.join(self._P1,
"bar.txt.in")).plugin_object, "text")
def __init__(self, name, version, description, secure, gettext_domain,
jobs_dir, whitelists_dir, data_dir, bin_dir, locale_dir,
base_dir):
"""
Initialize a provider with a set of meta-data and directories.
:param name:
provider name / ID
:param version:
provider version
:param description:
provider version
This is the untranslated version of this field. Implementations may
obtain the localized version based on the gettext_domain property.
:param secure:
secure bit
When True jobs from this provider should be available via the
trusted launcher mechanism. It should be set to True for
system-wide installed providers.
:param gettext_domain:
gettext domain that contains translations for this provider
:param jobs_dir:
path of the directory with job definitions
:param whitelists_dir:
path of the directory with whitelists definitions (aka test-plans)
:param data_dir:
path of the directory with files used by jobs at runtime
:param bin_dir:
path of the directory with additional executables
:param locale_dir:
path of the directory with locale database (translation catalogs)
:param base_dir:
path of the directory with (perhaps) all of jobs_dir,
whitelist_dir, data_dir, bin_dir, locale_dir. This may be None.
This is also the effective value of $CHECKBOX_SHARE
"""
# Meta-data
self._name = name
self._version = version
self._description = description
self._secure = secure
self._gettext_domain = gettext_domain
# Directories
self._jobs_dir = jobs_dir
self._whitelists_dir = whitelists_dir
self._data_dir = data_dir
self._bin_dir = bin_dir
self._locale_dir = locale_dir
self._base_dir = base_dir
# Loaded data
if self.whitelists_dir is not None:
whitelists_dir_list = [self.whitelists_dir]
else:
whitelists_dir_list = []
self._whitelist_collection = FsPlugInCollection(
whitelists_dir_list, ext=".whitelist", wrapper=WhiteListPlugIn,
implicit_namespace=self.namespace)
if self.jobs_dir is not None:
jobs_dir_list = [self.jobs_dir]
else:
jobs_dir_list = []
self._job_collection = FsPlugInCollection(
jobs_dir_list, ext=(".txt", ".txt.in"),
wrapper=JobDefinitionPlugIn, provider=self)
# Setup translations
if gettext_domain and locale_dir:
gettext.bindtextdomain(self._gettext_domain, self._locale_dir)
class Provider1(IProvider1, IProviderBackend1):
"""
A v1 provider implementation.
A provider is a container of jobs and whitelists. It provides additional
meta-data and knows about location of essential directories to both load
structured data and provide runtime information for job execution.
Providers are normally loaded with :class:`Provider1PlugIn`, due to the
number of fields involved in basic initialization.
"""
def __init__(self, name, version, description, secure, gettext_domain,
jobs_dir, whitelists_dir, data_dir, bin_dir, locale_dir,
base_dir):
"""
Initialize a provider with a set of meta-data and directories.
:param name:
provider name / ID
:param version:
provider version
:param description:
provider version
This is the untranslated version of this field. Implementations may
obtain the localized version based on the gettext_domain property.
:param secure:
secure bit
When True jobs from this provider should be available via the
trusted launcher mechanism. It should be set to True for
system-wide installed providers.
:param gettext_domain:
gettext domain that contains translations for this provider
:param jobs_dir:
path of the directory with job definitions
:param whitelists_dir:
path of the directory with whitelists definitions (aka test-plans)
:param data_dir:
path of the directory with files used by jobs at runtime
:param bin_dir:
path of the directory with additional executables
:param locale_dir:
path of the directory with locale database (translation catalogs)
:param base_dir:
path of the directory with (perhaps) all of jobs_dir,
whitelist_dir, data_dir, bin_dir, locale_dir. This may be None.
This is also the effective value of $CHECKBOX_SHARE
"""
# Meta-data
self._name = name
self._version = version
self._description = description
self._secure = secure
self._gettext_domain = gettext_domain
# Directories
self._jobs_dir = jobs_dir
self._whitelists_dir = whitelists_dir
self._data_dir = data_dir
self._bin_dir = bin_dir
self._locale_dir = locale_dir
self._base_dir = base_dir
# Loaded data
if self.whitelists_dir is not None:
whitelists_dir_list = [self.whitelists_dir]
else:
whitelists_dir_list = []
self._whitelist_collection = FsPlugInCollection(
whitelists_dir_list, ext=".whitelist", wrapper=WhiteListPlugIn,
implicit_namespace=self.namespace)
if self.jobs_dir is not None:
jobs_dir_list = [self.jobs_dir]
else:
jobs_dir_list = []
self._job_collection = FsPlugInCollection(
jobs_dir_list, ext=(".txt", ".txt.in"),
wrapper=JobDefinitionPlugIn, provider=self)
# Setup translations
if gettext_domain and locale_dir:
gettext.bindtextdomain(self._gettext_domain, self._locale_dir)
@classmethod
def from_definition(cls, definition, secure):
"""
Initialize a provider from Provider1Definition object
:param definition:
A Provider1Definition object to use as reference
:param secure:
Value of the secure flag. This cannot be expressed by a definition
object.
This method simplifies initialization of a Provider1 object where the
caller already has a Provider1Definition object. Depending on the value
of ``definition.location`` all of the directories are either None or
initialized to a *good* (typical) value relative to *location*
The only value that you may want to adjust, for working with source
providers, is *locale_dir*, by default it would be ``location/locale``
but ``manage.py i18n`` creates ``location/build/mo``
"""
logger.debug("Loading provider from definition %r", definition)
# Initialize the provider object
return cls(
definition.name, definition.version, definition.description,
secure, definition.effective_gettext_domain,
definition.effective_jobs_dir, definition.effective_whitelists_dir,
definition.effective_data_dir, definition.effective_bin_dir,
definition.effective_locale_dir, definition.location or None)
def __repr__(self):
return "<{} name:{!r}>".format(self.__class__.__name__, self.name)
@property
def name(self):
"""
name of this provider
"""
return self._name
@property
def namespace(self):
"""
namespace component of the provider name
This property defines the namespace in which all provider jobs are
defined in. Jobs within one namespace do not need to be fully qualified
by prefixing their partial identifier with provider namespace (so all
stays 'as-is'). Jobs that need to interact with other provider
namespaces need to use the fully qualified job identifier instead.
The identifier is defined as the part of the provider name, up to the
colon. This effectively gives organizations flat namespace within one
year-domain pair and allows to create private namespaces by using
sub-domains.
"""
return self._name.split(':', 1)[0]
@property
def version(self):
"""
version of this provider
"""
return self._version
@property
def description(self):
"""
description of this provider
"""
return self._description
def tr_description(self):
"""
Get the translated version of :meth:`description`
"""
return self.get_translated_data(self.description)
@property
def jobs_dir(self):
"""
absolute path of the jobs directory
"""
return self._jobs_dir
@property
def whitelists_dir(self):
"""
absolute path of the whitelist directory
"""
return self._whitelists_dir
@property
def data_dir(self):
"""
absolute path of the data directory
"""
return self._data_dir
@property
def bin_dir(self):
"""
absolute path of the bin directory
.. note::
The programs in that directory may not work without setting
PYTHONPATH and CHECKBOX_SHARE.
"""
return self._bin_dir
@property
def locale_dir(self):
"""
absolute path of the directory with locale data
The value is applicable as argument bindtextdomain()
"""
return self._locale_dir
@property
def base_dir(self):
"""
path of the directory with (perhaps) all of jobs_dir, whitelist_dir,
data_dir, bin_dir, locale_dir. This may be None
"""
return self._base_dir
@property
def CHECKBOX_SHARE(self):
"""
required value of CHECKBOX_SHARE environment variable.
.. note::
This variable is only required by one script.
It would be nice to remove this later on.
"""
return self.base_dir
@property
def extra_PYTHONPATH(self):
"""
additional entry for PYTHONPATH, if needed.
This entry is required for CheckBox scripts to import the correct
CheckBox python libraries.
.. note::
The result may be None
"""
return None
@property
def secure(self):
"""
flag indicating that this provider was loaded from the secure portion
of PROVIDERPATH and thus can be used with the
plainbox-trusted-launcher-1.
"""
return self._secure
@property
def gettext_domain(self):
"""
the name of the gettext domain associated with this provider
This value may be empty, in such case provider data cannot be localized
for the user environment.
"""
return self._gettext_domain
def get_builtin_whitelists(self):
"""
Load all the whitelists from :attr:`whitelists_dir` and return them
This method looks at the whitelist directory and loads all files ending
with .whitelist as a WhiteList object.
:returns:
A list of :class:`~plainbox.impl.secure.qualifiers.WhiteList`
objects sorted by
:attr:`plainbox.impl.secure.qualifiers.WhiteList.name`.
:raises IOError, OSError:
if there were any problems accessing files or directories. Note
that OSError is silently ignored when the `whitelists_dir`
directory is missing.
"""
self._whitelist_collection.load()
if self._whitelist_collection.problem_list:
raise self._whitelist_collection.problem_list[0]
else:
return sorted(self._whitelist_collection.get_all_plugin_objects(),
key=lambda whitelist: whitelist.name)
def get_builtin_jobs(self):
"""
Load and parse all of the job definitions of this provider.
:returns:
A sorted list of JobDefinition objects
:raises RFC822SyntaxError:
if any of the loaded files was not valid RFC822
:raises IOError, OSError:
if there were any problems accessing files or directories.
Note that OSError is silently ignored when the `jobs_dir`
directory is missing.
..note::
This method should not be used anymore. Consider transitioning your
code to :meth:`load_all_jobs()` which is more reliable.
"""
job_list, problem_list = self.load_all_jobs()
if problem_list:
raise problem_list[0]
else:
return job_list
def load_all_jobs(self):
"""
Load and parse all of the job definitions of this provider.
Unlike :meth:`get_builtin_jobs()` this method does not stop after the
first problem encountered and instead collects all of the problems into
a list which is returned alongside the job list.
:returns:
Pair (job_list, problem_list) where each job_list is a sorted list
of JobDefinition objects and each item from problem_list is an
exception.
"""
self._job_collection.load()
job_list = sorted(
itertools.chain(
*self._job_collection.get_all_plugin_objects()),
key=lambda job: job.id)
problem_list = self._job_collection.problem_list
return job_list, problem_list
def get_all_executables(self):
"""
Discover and return all executables offered by this provider
:returns:
list of executable names (without the full path)
:raises IOError, OSError:
if there were any problems accessing files or directories. Note
that OSError is silently ignored when the `bin_dir` directory is
missing.
"""
executable_list = []
if self.bin_dir is None:
return executable_list
try:
items = os.listdir(self.bin_dir)
except OSError as exc:
if exc.errno == errno.ENOENT:
items = []
else:
raise
for name in items:
filename = os.path.join(self.bin_dir, name)
if os.access(filename, os.F_OK | os.X_OK):
executable_list.append(filename)
return sorted(executable_list)
def get_translated_data(self, msgid):
"""
Get a localized piece of data
:param msgid:
data to translate
:returns:
translated data obtained from the provider if msgid is not False
(empty string and None both are) and this provider has a
gettext_domain defined for it, msgid itself otherwise.
"""
if msgid and self._gettext_domain:
return gettext.dgettext(self._gettext_domain, msgid)
else:
return msgid
class Provider1(IProvider1, IProviderBackend1):
"""
A v1 provider implementation.
This base class implements a checkbox-like provider object. Subclasses are
only required to implement a single method that designates the base
location for all other data.
"""
def __init__(self, base_dir, name, version, description, secure,
gettext_domain=None):
"""
Initialize the provider with the associated base directory.
All of the typical v1 provider data is relative to this directory. It
can be customized by subclassing and overriding the particular methods
of the IProviderBackend1 class but that should not be necessary in
normal operation.
"""
self._base_dir = base_dir
self._name = name
self._version = version
self._description = description
self._secure = secure
self._gettext_domain = gettext_domain
self._whitelist_collection = FsPlugInCollection(
[self.whitelists_dir], ext=".whitelist", wrapper=WhiteListPlugIn)
self._job_collection = FsPlugInCollection(
[self.jobs_dir], ext=(".txt", ".txt.in"),
wrapper=JobDefinitionPlugIn, provider=self)
def __repr__(self):
return "<{} name:{!r} base_dir:{!r}>".format(
self.__class__.__name__, self.name, self.base_dir)
@property
def base_dir(self):
"""
pathname to a directory with essential provider data
This pathname is used for deriving :attr:`jobs_dir`, :attr:`bin_dir`
and :attr:`whitelists_dir`.
"""
return self._base_dir
@property
def name(self):
"""
name of this provider
"""
return self._name
@property
def version(self):
"""
version of this provider
"""
return self._version
@property
def description(self):
"""
description of this provider
"""
return self._description
@property
def jobs_dir(self):
"""
Return an absolute path of the jobs directory
"""
return os.path.join(self._base_dir, "jobs")
@property
def bin_dir(self):
"""
Return an absolute path of the bin directory
.. note::
The programs in that directory may not work without setting
PYTHONPATH and CHECKBOX_SHARE.
"""
return os.path.join(self._base_dir, "bin")
@property
def whitelists_dir(self):
"""
Return an absolute path of the whitelist directory
"""
return os.path.join(self._base_dir, "whitelists")
@property
def CHECKBOX_SHARE(self):
"""
Return the required value of CHECKBOX_SHARE environment variable.
.. note::
This variable is only required by one script.
It would be nice to remove this later on.
"""
return self._base_dir
@property
def extra_PYTHONPATH(self):
"""
Return additional entry for PYTHONPATH, if needed.
This entry is required for CheckBox scripts to import the correct
CheckBox python libraries.
.. note::
The result may be None
"""
return None
@property
def secure(self):
"""
flag indicating that this provider was loaded from the secure portion
of PROVIDERPATH and thus can be used with the
plainbox-trusted-launcher-1.
"""
return self._secure
@property
def gettext_domain(self):
"""
the name of the gettext domain associated with this provider
This value may be empty, in such case provider data cannot be localized
for the user environment.
"""
return self._gettext_domain
def get_builtin_whitelists(self):
"""
Load all the whitelists from :attr:`whitelists_dir` and return them
This method looks at the whitelist directory and loads all files ending
with .whitelist as a WhiteList object.
:returns:
A list of :class:`~plainbox.impl.secure.qualifiers.WhiteList`
objects sorted by
:attr:`plainbox.impl.secure.qualifiers.WhiteList.name`.
:raises IOError, OSError:
if there were any problems accessing files or directories. Note
that OSError is silently ignored when the `whitelists_dir`
directory is missing.
"""
self._whitelist_collection.load()
if self._whitelist_collection.problem_list:
raise self._whitelist_collection.problem_list[0]
else:
return sorted(self._whitelist_collection.get_all_plugin_objects(),
key=lambda whitelist: whitelist.name)
def get_builtin_jobs(self):
"""
Load and parse all of the job definitions of this provider.
:returns:
A sorted list of JobDefinition objects
:raises RFC822SyntaxError:
if any of the loaded files was not valid RFC822
:raises IOError, OSError:
if there were any problems accessing files or directories.
Note that OSError is silently ignored when the `jobs_dir`
directory is missing.
..note::
This method should not be used anymore. Consider transitioning your
code to :meth:`load_all_jobs()` which is more reliable.
"""
job_list, problem_list = self.load_all_jobs()
if problem_list:
raise problem_list[0]
else:
return job_list
def load_all_jobs(self):
"""
Load and parse all of the job definitions of this provider.
Unlike :meth:`get_builtin_jobs()` this method does not stop after the
first problem encountered and instead collects all of the problems into
a list which is returned alongside the job list.
:returns:
Pair (job_list, problem_list) where each job_list is a sorted list
of JobDefinition objects and each item from problem_list is an
exception.
"""
self._job_collection.load()
job_list = sorted(
itertools.chain(
*self._job_collection.get_all_plugin_objects()),
key=lambda job: job.name)
problem_list = self._job_collection.problem_list
return job_list, problem_list
def get_all_executables(self):
"""
Discover and return all executables offered by this provider
:returns:
list of executable names (without the full path)
:raises IOError, OSError:
if there were any problems accessing files or directories. Note
that OSError is silently ignored when the `bin_dir` directory is
missing.
"""
executable_list = []
try:
items = os.listdir(self.bin_dir)
except OSError as exc:
if exc.errno == errno.ENOENT:
items = []
else:
raise
for name in items:
filename = os.path.join(self.bin_dir, name)
if os.access(filename, os.F_OK | os.X_OK):
executable_list.append(filename)
return sorted(executable_list)
def setUp(self):
# Create a collection
self.col = FsPlugInCollection(self._DIR_LIST, self._EXT)
class FsPlugInCollectionTests(TestCase):
_P1 = "/system/providers"
_P2 = "home/user/.providers"
_DIR_LIST = [_P1, _P2]
_EXT = ".plugin"
def setUp(self):
# Create a collection
self.col = FsPlugInCollection(self._DIR_LIST, self._EXT)
def test_path_is_set(self):
# Ensure that path was saved
self.assertEqual(self.col._dir_list, self._DIR_LIST)
def test_ext_is_set(self):
# Ensure that ext was saved
self.assertEqual(self.col._ext, self._EXT)
def test_plugins_are_empty(self):
# Ensure that plugins start out empty
self.assertEqual(len(self.col._plugins), 0)
def test_initial_loaded_flag(self):
# Ensure that 'loaded' flag is false
self.assertFalse(self.col._loaded)
def test_default_wrapper(self):
# Ensure that the wrapper is :class:`PlugIn`
self.assertEqual(self.col._wrapper, PlugIn)
@mock.patch('plainbox.impl.secure.plugins.logger')
@mock.patch('builtins.open')
@mock.patch('os.path.isfile')
@mock.patch('os.listdir')
def test_load(self, mock_listdir, mock_isfile, mock_open, mock_logger):
# Mock a bit of filesystem access methods to make some plugins show up
def fake_listdir(path):
if path == self._P1:
return [
# A regular plugin
'foo.plugin',
# Another regular plugin
'bar.plugin',
# Unrelated file, not a plugin
'unrelated.txt',
# A directory that looks like a plugin
'dir.bad.plugin',
# A plugin without read permissions
'noperm.plugin']
else:
raise OSError("There is nothing in {}".format(path))
def fake_isfile(path):
return not os.path.basename(path).startswith('dir.')
def fake_open(path, encoding=None, mode=None):
m = mock.MagicMock(name='opened file {!r}'.format(path))
m.__enter__.return_value = m
if path == os.path.join(self._P1, 'foo.plugin'):
m.read.return_value = "foo"
return m
elif path == os.path.join(self._P1, 'bar.plugin'):
m.read.return_value = "bar"
return m
elif path == os.path.join(self._P1, 'noperm.plugin'):
raise OSError("You cannot open this file")
else:
raise IOError("Unexpected file: {}".format(path))
mock_listdir.side_effect = fake_listdir
mock_isfile.side_effect = fake_isfile
mock_open.side_effect = fake_open
# Load all plugins now
self.col.load()
# And 'again', just to ensure we're doing the IO only once
self.col.load()
# Ensure that we actually tried to look at the filesytstem
self.assertEqual(
mock_listdir.call_args_list, [
((self._P1, ), {}),
((self._P2, ), {})
])
# Ensure that we actually tried to check if things are files
self.assertEqual(
mock_isfile.call_args_list, [
((os.path.join(self._P1, 'foo.plugin'),), {}),
((os.path.join(self._P1, 'bar.plugin'),), {}),
((os.path.join(self._P1, 'dir.bad.plugin'),), {}),
((os.path.join(self._P1, 'noperm.plugin'),), {}),
])
# Ensure that we actually tried to open some files
self.assertEqual(
mock_open.call_args_list, [
((os.path.join(self._P1, 'bar.plugin'),),
{'encoding': 'UTF-8'}),
((os.path.join(self._P1, 'foo.plugin'),),
{'encoding': 'UTF-8'}),
((os.path.join(self._P1, 'noperm.plugin'),),
{'encoding': 'UTF-8'}),
])
# Ensure that an exception was logged
mock_logger.error.assert_called_with(
'Unable to load %r: %s',
'/system/providers/noperm.plugin',
'You cannot open this file')
# Ensure that all of the errors are collected
# Using repr() since OSError seems hard to compare correctly
self.assertEqual(
repr(self.col.problem_list[0]),
repr(OSError('You cannot open this file')))
@mock.patch('plainbox.impl.secure.plugins.logger')
@mock.patch('builtins.open')
@mock.patch('os.path.isfile')
@mock.patch('os.listdir')
def test_load__two_extensions(self, mock_listdir, mock_isfile, mock_open,
mock_logger):
"""
verify that FsPlugInCollection works with multiple extensions
"""
mock_listdir.return_value = ["foo.txt", "bar.txt.in"]
mock_isfile.return_value = True
def fake_open(path, encoding=None, mode=None):
m = mock.MagicMock(name='opened file {!r}'.format(path))
m.read.return_value = "text"
m.__enter__.return_value = m
return m
mock_open.side_effect = fake_open
# Create a collection that looks for both extensions
col = FsPlugInCollection([self._P1], (".txt", ".txt.in"))
# Load everything
col.load()
# Ensure that we actually tried to look at the filesystem
self.assertEqual(
mock_listdir.call_args_list, [
((self._P1, ), {}),
])
# Ensure that we actually tried to check if things are files
self.assertEqual(
mock_isfile.call_args_list, [
((os.path.join(self._P1, 'foo.txt'),), {}),
((os.path.join(self._P1, 'bar.txt.in'),), {}),
])
# Ensure that we actually tried to open some files
self.assertEqual(
mock_open.call_args_list, [
((os.path.join(self._P1, 'bar.txt.in'),),
{'encoding': 'UTF-8'}),
((os.path.join(self._P1, 'foo.txt'),),
{'encoding': 'UTF-8'}),
])
# Ensure that no exception was logged
mock_logger.error.assert_not_called()
# Ensure that everything was okay
self.assertEqual(col.problem_list, [])
# Ensure that both files got added
self.assertEqual(
col.get_by_name(
os.path.join(self._P1, "foo.txt")
).plugin_object, "text")
self.assertEqual(
col.get_by_name(
os.path.join(self._P1, "bar.txt.in")
).plugin_object, "text")
def __init__(self, name, version, description, secure, gettext_domain,
jobs_dir, whitelists_dir, data_dir, bin_dir, locale_dir,
base_dir):
"""
Initialize a provider with a set of meta-data and directories.
:param name:
provider name / ID
:param version:
provider version
:param description:
provider version
This is the untranslated version of this field. Implementations may
obtain the localized version based on the gettext_domain property.
:param secure:
secure bit
When True jobs from this provider should be available via the
trusted launcher mechanism. It should be set to True for
system-wide installed providers.
:param gettext_domain:
gettext domain that contains translations for this provider
:param jobs_dir:
path of the directory with job definitions
:param whitelists_dir:
path of the directory with whitelists definitions (aka test-plans)
:param data_dir:
path of the directory with files used by jobs at runtime
:param bin_dir:
path of the directory with additional executables
:param locale_dir:
path of the directory with locale database (translation catalogs)
:param base_dir:
path of the directory with (perhaps) all of jobs_dir,
whitelist_dir, data_dir, bin_dir, locale_dir. This may be None.
This is also the effective value of $CHECKBOX_SHARE
"""
# Meta-data
self._name = name
self._version = version
self._description = description
self._secure = secure
self._gettext_domain = gettext_domain
# Directories
self._jobs_dir = jobs_dir
self._whitelists_dir = whitelists_dir
self._data_dir = data_dir
self._bin_dir = bin_dir
self._locale_dir = locale_dir
self._base_dir = base_dir
# Loaded data
if self.whitelists_dir is not None:
whitelists_dir_list = [self.whitelists_dir]
else:
whitelists_dir_list = []
self._whitelist_collection = FsPlugInCollection(
whitelists_dir_list,
ext=".whitelist",
wrapper=WhiteListPlugIn,
implicit_namespace=self.namespace)
if self.jobs_dir is not None:
jobs_dir_list = [self.jobs_dir]
else:
jobs_dir_list = []
self._job_collection = FsPlugInCollection(jobs_dir_list,
ext=(".txt", ".txt.in"),
wrapper=JobDefinitionPlugIn,
provider=self)
# Setup translations
if gettext_domain and locale_dir:
gettext.bindtextdomain(self._gettext_domain, self._locale_dir)
class Provider1(IProvider1, IProviderBackend1):
"""
A v1 provider implementation.
A provider is a container of jobs and whitelists. It provides additional
meta-data and knows about location of essential directories to both load
structured data and provide runtime information for job execution.
Providers are normally loaded with :class:`Provider1PlugIn`, due to the
number of fields involved in basic initialization.
"""
def __init__(self, name, version, description, secure, gettext_domain,
jobs_dir, whitelists_dir, data_dir, bin_dir, locale_dir,
base_dir):
"""
Initialize a provider with a set of meta-data and directories.
:param name:
provider name / ID
:param version:
provider version
:param description:
provider version
This is the untranslated version of this field. Implementations may
obtain the localized version based on the gettext_domain property.
:param secure:
secure bit
When True jobs from this provider should be available via the
trusted launcher mechanism. It should be set to True for
system-wide installed providers.
:param gettext_domain:
gettext domain that contains translations for this provider
:param jobs_dir:
path of the directory with job definitions
:param whitelists_dir:
path of the directory with whitelists definitions (aka test-plans)
:param data_dir:
path of the directory with files used by jobs at runtime
:param bin_dir:
path of the directory with additional executables
:param locale_dir:
path of the directory with locale database (translation catalogs)
:param base_dir:
path of the directory with (perhaps) all of jobs_dir,
whitelist_dir, data_dir, bin_dir, locale_dir. This may be None.
This is also the effective value of $CHECKBOX_SHARE
"""
# Meta-data
self._name = name
self._version = version
self._description = description
self._secure = secure
self._gettext_domain = gettext_domain
# Directories
self._jobs_dir = jobs_dir
self._whitelists_dir = whitelists_dir
self._data_dir = data_dir
self._bin_dir = bin_dir
self._locale_dir = locale_dir
self._base_dir = base_dir
# Loaded data
if self.whitelists_dir is not None:
whitelists_dir_list = [self.whitelists_dir]
else:
whitelists_dir_list = []
self._whitelist_collection = FsPlugInCollection(
whitelists_dir_list,
ext=".whitelist",
wrapper=WhiteListPlugIn,
implicit_namespace=self.namespace)
if self.jobs_dir is not None:
jobs_dir_list = [self.jobs_dir]
else:
jobs_dir_list = []
self._job_collection = FsPlugInCollection(jobs_dir_list,
ext=(".txt", ".txt.in"),
wrapper=JobDefinitionPlugIn,
provider=self)
# Setup translations
if gettext_domain and locale_dir:
gettext.bindtextdomain(self._gettext_domain, self._locale_dir)
@classmethod
def from_definition(cls, definition, secure):
"""
Initialize a provider from Provider1Definition object
:param definition:
A Provider1Definition object to use as reference
:param secure:
Value of the secure flag. This cannot be expressed by a definition
object.
This method simplifies initialization of a Provider1 object where the
caller already has a Provider1Definition object. Depending on the value
of ``definition.location`` all of the directories are either None or
initialized to a *good* (typical) value relative to *location*
The only value that you may want to adjust, for working with source
providers, is *locale_dir*, by default it would be ``location/locale``
but ``manage.py i18n`` creates ``location/build/mo``
"""
logger.debug("Loading provider from definition %r", definition)
# Initialize the provider object
return cls(definition.name, definition.version, definition.description,
secure, definition.effective_gettext_domain,
definition.effective_jobs_dir,
definition.effective_whitelists_dir,
definition.effective_data_dir, definition.effective_bin_dir,
definition.effective_locale_dir, definition.location
or None)
def __repr__(self):
return "<{} name:{!r}>".format(self.__class__.__name__, self.name)
@property
def name(self):
"""
name of this provider
"""
return self._name
@property
def namespace(self):
"""
namespace component of the provider name
This property defines the namespace in which all provider jobs are
defined in. Jobs within one namespace do not need to be fully qualified
by prefixing their partial identifier with provider namespace (so all
stays 'as-is'). Jobs that need to interact with other provider
namespaces need to use the fully qualified job identifier instead.
The identifier is defined as the part of the provider name, up to the
colon. This effectively gives organizations flat namespace within one
year-domain pair and allows to create private namespaces by using
sub-domains.
"""
return self._name.split(':', 1)[0]
@property
def version(self):
"""
version of this provider
"""
return self._version
@property
def description(self):
"""
description of this provider
"""
return self._description
def tr_description(self):
"""
Get the translated version of :meth:`description`
"""
return self.get_translated_data(self.description)
@property
def jobs_dir(self):
"""
absolute path of the jobs directory
"""
return self._jobs_dir
@property
def whitelists_dir(self):
"""
absolute path of the whitelist directory
"""
return self._whitelists_dir
@property
def data_dir(self):
"""
absolute path of the data directory
"""
return self._data_dir
@property
def bin_dir(self):
"""
absolute path of the bin directory
.. note::
The programs in that directory may not work without setting
PYTHONPATH and CHECKBOX_SHARE.
"""
return self._bin_dir
@property
def locale_dir(self):
"""
absolute path of the directory with locale data
The value is applicable as argument bindtextdomain()
"""
return self._locale_dir
@property
def base_dir(self):
"""
path of the directory with (perhaps) all of jobs_dir, whitelist_dir,
data_dir, bin_dir, locale_dir. This may be None
"""
return self._base_dir
@property
def CHECKBOX_SHARE(self):
"""
required value of CHECKBOX_SHARE environment variable.
.. note::
This variable is only required by one script.
It would be nice to remove this later on.
"""
return self.base_dir
@property
def extra_PYTHONPATH(self):
"""
additional entry for PYTHONPATH, if needed.
This entry is required for CheckBox scripts to import the correct
CheckBox python libraries.
.. note::
The result may be None
"""
return None
@property
def secure(self):
"""
flag indicating that this provider was loaded from the secure portion
of PROVIDERPATH and thus can be used with the
plainbox-trusted-launcher-1.
"""
return self._secure
@property
def gettext_domain(self):
"""
the name of the gettext domain associated with this provider
This value may be empty, in such case provider data cannot be localized
for the user environment.
"""
return self._gettext_domain
def get_builtin_whitelists(self):
"""
Load all the whitelists from :attr:`whitelists_dir` and return them
This method looks at the whitelist directory and loads all files ending
with .whitelist as a WhiteList object.
:returns:
A list of :class:`~plainbox.impl.secure.qualifiers.WhiteList`
objects sorted by
:attr:`plainbox.impl.secure.qualifiers.WhiteList.name`.
:raises IOError, OSError:
if there were any problems accessing files or directories. Note
that OSError is silently ignored when the `whitelists_dir`
directory is missing.
"""
self._whitelist_collection.load()
if self._whitelist_collection.problem_list:
raise self._whitelist_collection.problem_list[0]
else:
return sorted(self._whitelist_collection.get_all_plugin_objects(),
key=lambda whitelist: whitelist.name)
def get_builtin_jobs(self):
"""
Load and parse all of the job definitions of this provider.
:returns:
A sorted list of JobDefinition objects
:raises RFC822SyntaxError:
if any of the loaded files was not valid RFC822
:raises IOError, OSError:
if there were any problems accessing files or directories.
Note that OSError is silently ignored when the `jobs_dir`
directory is missing.
..note::
This method should not be used anymore. Consider transitioning your
code to :meth:`load_all_jobs()` which is more reliable.
"""
job_list, problem_list = self.load_all_jobs()
if problem_list:
raise problem_list[0]
else:
return job_list
def load_all_jobs(self):
"""
Load and parse all of the job definitions of this provider.
Unlike :meth:`get_builtin_jobs()` this method does not stop after the
first problem encountered and instead collects all of the problems into
a list which is returned alongside the job list.
:returns:
Pair (job_list, problem_list) where each job_list is a sorted list
of JobDefinition objects and each item from problem_list is an
exception.
"""
self._job_collection.load()
job_list = sorted(
itertools.chain(*self._job_collection.get_all_plugin_objects()),
key=lambda job: job.id)
problem_list = self._job_collection.problem_list
return job_list, problem_list
def get_all_executables(self):
"""
Discover and return all executables offered by this provider
:returns:
list of executable names (without the full path)
:raises IOError, OSError:
if there were any problems accessing files or directories. Note
that OSError is silently ignored when the `bin_dir` directory is
missing.
"""
executable_list = []
if self.bin_dir is None:
return executable_list
try:
items = os.listdir(self.bin_dir)
except OSError as exc:
if exc.errno == errno.ENOENT:
items = []
else:
raise
for name in items:
filename = os.path.join(self.bin_dir, name)
if os.access(filename, os.F_OK | os.X_OK):
executable_list.append(filename)
return sorted(executable_list)
def get_translated_data(self, msgid):
"""
Get a localized piece of data
:param msgid:
data to translate
:returns:
translated data obtained from the provider if msgid is not False
(empty string and None both are) and this provider has a
gettext_domain defined for it, msgid itself otherwise.
"""
if msgid and self._gettext_domain:
return gettext.dgettext(self._gettext_domain, msgid)
else:
return msgid
