def test_inifile_nonexistent(self):
logging.getLogger().setLevel(logging.CRITICAL)
cfg = LayeredConfig(INIFile("nonexistent.ini"))
self.assertEqual([], list(cfg))
# make sure a nonexistent inifile doesn't interfere with the
# rest of the LayeredConfig object
defobj = Defaults({'datadir': 'something'})
iniobj = INIFile("nonexistent.ini")
cfg = LayeredConfig(defobj, iniobj)
self.assertEqual("something", cfg.datadir)
# and make sure it's settable (should set up the INIFile
# object and affect it, and leave the defaults dict untouched
# as it's the lowest priority)
cfg.datadir = "else"
self.assertEqual("else", cfg.datadir)
self.assertEqual("else", iniobj.get("datadir"))
self.assertEqual("something", defobj.get("datadir"))
# same as above, but with a "empty" INIFile object
iniobj = INIFile()
cfg = LayeredConfig(defobj, iniobj)
self.assertEqual("something", cfg.datadir)
cfg.datadir = "else"
self.assertEqual("else", cfg.datadir)
def test_layered_subsections(self):
defaults = OrderedDict(
(('force', False), ('home', 'thisdata'), ('loglevel', 'INFO')))
cmdline = ['--mymodule-home=thatdata', '--mymodule-force']
cfg = LayeredConfig(Defaults(defaults),
Commandline(cmdline),
cascade=True)
self.assertEqual(cfg.mymodule.force, True)
self.assertEqual(cfg.mymodule.home, 'thatdata')
self.assertEqual(cfg.mymodule.loglevel, 'INFO')
# second test is more difficult: the lower-priority Defaults
# source only contains a subsection, while the higher-priority
# Commandline source contains no such subsection. Our
# sub-LayeredConfig object will only have a Defaults source,
# not a Commandline source (which will cause the
# __getattribute__ lookup_resource to look in the Defaults
# object in the sub-LayeredConfig object, unless we do
# something smart.
defaults = {'mymodule': defaults}
cmdline = ['--home=thatdata', '--force']
o = Commandline(cmdline)
o.subsection("mymodule").keys()
cfg = LayeredConfig(Defaults(defaults),
Commandline(cmdline),
cascade=True)
self.assertEqual(cfg.mymodule.force, True)
self.assertEqual(cfg.mymodule.home, 'thatdata')
self.assertEqual(cfg.mymodule.loglevel, 'INFO')
self.assertEqual(['force', 'home', 'loglevel'], list(cfg.mymodule))
def test_commandline_implicit_typing(self):
# The big test here is really the partially-configured
# ArgumentParser (handles one positional argument but not the
# optional --force)
defaults = {'force': False}
cmdline = ['command', '--force']
parser = argparse.ArgumentParser()
parser.add_argument("positional")
cmdlinesrc = Commandline(cmdline, parser=parser)
cfg = LayeredConfig(Defaults(defaults), cmdlinesrc)
self.assertEqual(cfg.force, True)
# try again with explicit argument
parser = argparse.ArgumentParser()
parser.add_argument("positional")
cmdlinesrc = Commandline(['command', '--force=True'], parser=parser)
cfg = LayeredConfig(Defaults(defaults), cmdlinesrc)
self.assertEqual(cfg.force, True)
# once again without the optional typing source
parser = argparse.ArgumentParser()
parser.add_argument("positional")
cmdlinesrc = Commandline(['command', '--force'], parser=parser)
cfg = LayeredConfig(Defaults({}), cmdlinesrc)
self.assertEqual(cfg.force, True)
def test_queryindex(self):
res = [{
'label': 'Doc #1',
'uri': 'http://example.org/doc1',
'text': 'matching doc 1'
}, {
'label': 'Doc #2',
'uri': 'http://example.org/doc2',
'text': 'matching doc 2'
}]
pager = None
config = {
'connect.return_value':
Mock(**{'query.return_value': (res, pager)})
}
printmock = MagicMock()
with patch('ferenda.devel.FulltextIndex', **config):
with patch('builtins.print', printmock):
d = Devel()
d.config = LayeredConfig(
Defaults({
'indextype': 'a',
'indexlocation': 'b'
}))
d.queryindex("doc")
want = """
Doc #1 (http://example.org/doc1): matching doc 1
Doc #2 (http://example.org/doc2): matching doc 2
""".strip()
got = "\n".join([x[1][0] for x in printmock.mock_calls])
self.maxDiff = None
self.assertEqual(want, got)
def test_read_layered_sources_with_strategies():
config = LayeredConfig(
DictSource({
'a': 1,
'x': [5, 6],
'b': {
'c': 2,
'd': [3, 4]
}
}),
DictSource({
'a': 10,
'x': [50, 60],
'b': {
'c': 20,
'd': [30, 40]
}
}),
strategies={
'a': strategy.add,
'x': strategy.collect, # keep lists intact
'c': strategy.collect, # collect values into list
'd': strategy.merge, # merge lists
})
assert config.a == 11
assert config.x == [[50, 60], [5, 6]]
assert config.b.c == [20, 2]
assert config.b.d == [30, 40, 3, 4]
def __init__(self, repos, inifile=None, **kwargs):
self.repos = repos
self.log = logging.getLogger("wsgi")
# FIXME: Cut-n-paste of the method in Resources.__init__
loadpaths = [ResourceLoader.make_loadpath(repo) for repo in repos]
loadpath = ["."] # cwd always has priority -- makes sense?
for subpath in loadpaths:
for p in subpath:
if p not in loadpath:
loadpath.append(p)
self.resourceloader = ResourceLoader(*loadpath)
# FIXME: need to specify documentroot?
defaults = DocumentRepository.get_default_options()
if inifile:
assert os.path.exists(
inifile), "INI file %s doesn't exist (relative to %s)" % (
inifile, os.getcwd())
# NB: If both inifile and kwargs are specified, the latter
# will take precedence. I think this is the expected
# behaviour.
self.config = LayeredConfig(Defaults(defaults),
INIFile(inifile),
Defaults(kwargs),
cascade=True)
def test_write(self):
self.maxDiff = None
cfg = LayeredConfig(self.complex)
cfg.mymodule.expires = date(2014, 10, 24)
# calling write for any submodule will force a write of the
# entire config file
LayeredConfig.write(cfg.mymodule)
want = """{
"extra": [
"foo",
"bar"
],
"extramodule": {
"unique": true
},
"force": true,
"home": "mydata",
"mymodule": {
"arbitrary": {
"nesting": {
"depth": "works"
}
},
"expires": "2014-10-24",
"extra": [
"foo",
"baz"
],
"force": false
},
"processes": 4
}"""
with open("complex.json") as fp:
got = fp.read().replace("\r\n", "\n")
self.assertEqual(want, got)
def setUp(self):
super(WSGI,self).setUp()
if self.storelocation.startswith("data/"):
self.storelocation = self.storelocation.replace("data", self.datadir)
if self.indexlocation.startswith("data/"):
self.indexlocation = self.indexlocation.replace("data", self.datadir)
self.put_files_in_place()
# use self.repo (simple testcases) or self.repos (complex
# testcases like AdvancedAPI)?
if hasattr(self, 'repos'):
repos = self.repos
else:
repos = [self.repo]
# print("making app: %s %s" % (self.storetype, self.indextype))
config = LayeredConfig(Defaults({'datadir': self.datadir,
'apiendpoint': '/myapi/',
'searchendpoint': '/mysearch/',
'url': 'http://localhost:8000/',
'storetype': self.storetype,
'storelocation': self.storelocation,
'storerepository': self.storerepository,
'indextype': self.indextype,
'indexlocation': self.indexlocation,
'wsgiappclass': 'ferenda.WSGIApp',
'legacyapi': False,
'wsgiexceptionhandler': True}))
self.app = manager.make_wsgi_app(config, repos=repos)
self.builder = EnvironBuilder('/', base_url="http://localhost:8000/",
headers={"Accept": DEFAULT_HTTP_ACCEPT})
def _createrepo(self):
clsdefaults = self.repoclass.get_default_options()
configfile = self.datadir + os.sep + "ferenda.ini"
config = LayeredConfig(Defaults(clsdefaults),
INIFile(configfile),
cascade=True)
return self.repoclass(config)
def test_inifile_default_as_root(self):
# using a rootsection named DEFAULT triggers different
# cascading-like behaviour in configparser.
# load a modified version of complex.ini
with open("complex.ini") as fp:
ini = fp.read()
with open("complex-otherroot.ini", "w") as fp:
fp.write(ini.replace("[__root__]", "[DEFAULT]"))
cfg = LayeredConfig(
INIFile("complex-otherroot.ini", rootsection="DEFAULT"))
# this is a modified/simplified version of ._test_subsections
self.assertEqual(cfg.home, 'mydata')
self.assertEqual(cfg.processes, '4')
self.assertEqual(cfg.force, 'True')
self.assertEqual(cfg.mymodule.force, 'False')
self.assertEqual(cfg.extra, "foo, bar")
self.assertEqual(cfg.mymodule.extra, "foo, baz")
with self.assertRaises(AttributeError):
cfg.expires
self.assertEqual(cfg.mymodule.expires, "2014-10-15")
# this is really unwanted cascading behaviour
self.assertEqual(cfg.mymodule.home, 'mydata')
self.assertEqual(cfg.mymodule.processes, '4')
os.unlink("complex-otherroot.ini")
def __init__(self, repos, resourcedir, **kwargs):
# FIXME: document what kwargs could be (particularly 'combineresources')
self.repos = repos
self.resourcedir = resourcedir
from ferenda.manager import DEFAULT_CONFIG
defaults = dict(DEFAULT_CONFIG)
defaults.update(DocumentRepository.get_default_options())
defaults.update(kwargs)
self.config = LayeredConfig(Defaults(defaults))
# the below call to setup_logger alters the logging level of
# the root logger, which can't be good practice. Also, we
# should probably not log to the root logger, but rather to
# ferenda.resources.
#
# from ferenda.manager import setup_logger
# self.log = setup_logger()
self.log = logging.getLogger("ferenda.resources")
# FIXME: How should we set up a global loadpath from the
# individual repos?
loadpaths = [ResourceLoader.make_loadpath(repo) for repo in repos]
loadpath = ["."] # cwd always has priority -- makes sense?
for subpath in loadpaths:
for p in subpath:
if p not in loadpath:
loadpath.append(p)
self.resourceloader = ResourceLoader(*loadpath)
def test_write_layered_source():
source1 = DictSource({'a': 1, 'b': {'c': 2}})
source2 = DictSource({'x': 6, 'b': {'y': 7, 'd': {'e': 8}}})
config = LayeredConfig(source1, source2)
assert config.a == 1
assert config.b.c == 2
assert config.b.y == 7
config.a = 10
config['x'] = 60
config['b'].c = 20
config.b['y'] = 70
config.b['m'] = 'n' # add new key
config.b.d.e = 80
assert config.a == 10
assert config.x == 60
assert config.b.c == 20
assert config.b.y == 70
assert config.b.m == 'n'
assert config.b.d.e == 80
assert source1.a == 10
assert source1.b.c == 20
assert source2.x == 60
assert source2.b.y == 70
assert source2.b.m == 'n'
assert source2.b.d.e == 80
def test_write(self):
cfg = LayeredConfig(self.complex)
cfg.mymodule.expires = date(2014, 10, 24)
# calling write for any submodule will force a write of the
# entire config file
LayeredConfig.write(cfg.mymodule)
# note that pyyaml sorts keys alphabetically and has specific
# ideas on how to format the result (controllable through
# mostly-undocumented args to dump())
want = """
extra:
- foo
- bar
extramodule:
unique: true
force: true
home: mydata
mymodule:
arbitrary:
nesting:
depth: works
expires: 2014-10-24
extra:
- foo
- baz
force: false
processes: 4
""".lstrip()
with open("complex.yaml") as fp:
got = fp.read().replace("\r\n", "\n")
self.assertEqual(want, got)
def test_modified_subsections(self):
defaults = {'force': False, 'home': 'thisdata', 'loglevel': 'INFO'}
cmdline = ['--mymodule-home=thatdata', '--mymodule-force']
cfg = LayeredConfig(Defaults(defaults),
INIFile("complex.ini"),
Commandline(cmdline),
cascade=True)
cfg.mymodule.expires = date(2014, 10, 24)
def test_write_layered_source_fails(key, message):
source1 = DictSource({'a': 1, 'b': {'c': 2}}, readonly=True)
config = LayeredConfig(source1)
with pytest.raises(TypeError) as exc_info:
config[key] = 10
assert message in str(exc_info.value)
def test_select(self):
uri = "http://example.org/doc"
with open("testselecttemplate.rq", "wb") as fp:
fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/>
SELECT ?p ?o
WHERE { <%(uri)s> ?p ?o . }
""".encode())
result = """
[
{
"p": "http://purl.org/dc/terms/title",
"o": "Document title"
},
{
"p": "http://purl.org/dc/terms/identifier",
"o": "Document ID"
}
]""".lstrip().encode("utf-8")
config = {
'connect.return_value': Mock(**{'select.return_value': result})
}
printmock = MagicMock()
with patch('ferenda.devel.TripleStore', **config):
with patch('builtins.print', printmock):
d = Devel()
d.config = LayeredConfig(
Defaults({
'storetype': 'a',
'storelocation': 'b',
'storerepository': 'c'
}))
d.select("testselecttemplate.rq", uri)
want = """
# Constructing the following from b, repository c, type a
# PREFIX dcterms: <http://purl.org/dc/terms/>
#
# SELECT ?p ?o
# WHERE { <http://example.org/doc> ?p ?o . }
#
[
{
"p": "http://purl.org/dc/terms/title",
"o": "Document title"
},
{
"p": "http://purl.org/dc/terms/identifier",
"o": "Document ID"
}
]
# Selected in 0.001s
""".strip()
got = "\n".join([x[1][0] for x in printmock.mock_calls])
self.maxDiff = None
self.assertEqual(self.mask_time(want), self.mask_time(got))
os.unlink("testselecttemplate.rq")
def test_layered(self):
defaults = {'home': 'someplace'}
cmdline = ['--home=anotherplace']
env = {'MYAPP_HOME': 'yourdata'}
cfg = LayeredConfig(Defaults(defaults))
self.assertEqual(cfg.home, 'someplace')
cfg = LayeredConfig(Defaults(defaults), INIFile("simple.ini"))
self.assertEqual(cfg.home, 'mydata')
cfg = LayeredConfig(Defaults(defaults), INIFile("simple.ini"),
Environment(env, prefix="MYAPP_"))
self.assertEqual(cfg.home, 'yourdata')
cfg = LayeredConfig(Defaults(defaults), INIFile("simple.ini"),
Environment(env, prefix="MYAPP_"),
Commandline(cmdline))
self.assertEqual(cfg.home, 'anotherplace')
self.assertEqual(
['home', 'processes', 'force', 'extra', 'expires', 'lastrun'],
list(cfg))
def test_write(self):
self.maxDiff = None
cfg = LayeredConfig(self.complex)
cfg.mymodule.expires = date(2014, 10, 24)
# calling write for any submodule will force a write of the
# entire config file
LayeredConfig.write(cfg.mymodule)
# note: plistlib creates files with tabs, not spaces.
want = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>extra</key>
<array>
<string>foo</string>
<string>bar</string>
</array>
<key>extramodule</key>
<dict>
<key>unique</key>
<true/>
</dict>
<key>force</key>
<true/>
<key>home</key>
<string>mydata</string>
<key>mymodule</key>
<dict>
<key>arbitrary</key>
<dict>
<key>nesting</key>
<dict>
<key>depth</key>
<string>works</string>
</dict>
</dict>
<key>expires</key>
<string>2014-10-24</string>
<key>extra</key>
<array>
<string>foo</string>
<string>baz</string>
</array>
<key>force</key>
<false/>
</dict>
<key>processes</key>
<integer>4</integer>
</dict>
</plist>
"""
if sys.version_info < (2, 7, 0): # pragma: no cover
# on py26, the doctype includes "Apple Computer" not "Apple"...
want = want.replace("//Apple//", "//Apple Computer//")
with open("complex.plist") as fp:
got = fp.read().replace("\r\n", "\n")
self.assertEqual(want, got)
def test_set_keychain():
config = LayeredConfig(DictSource({'a': {
'b': {
'c': 2
}
}}),
keychain=('a', 'b'))
assert config.dump() == {'c': 2}
def test_read_complex_layered_sources(monkeypatch):
monkeypatch.setenv('MVP1_A', 1000)
monkeypatch.setenv('MVP2_B_M_E', 4000)
config = LayeredConfig(
Environment('MVP1_'), # untyped shadowing
DictSource({
'a': 1,
'b': {
'c': 2,
'e': 400
}
}),
DictSource({
'x': 6,
'b': {
'y': 7,
'd': {
'e': 8
}
}
}),
DictSource({
'a': 100,
'b': {
'm': {
'e': 800
}
}
}), # shadowing
DictSource({
'x': 'x',
'b': {
'y': 0.7,
'd': 800
}
}), # type changing
Environment('MVP2_'), # untyped shadowing
)
assert config.a == 100
assert config.x == 'x' # changes int to str
assert config.b.c == 2
assert config.b.y == 0.7 # changes int to float
assert config.b.d == 800 # changes subsource (dict) to single value
assert config.b.e == 400 # 'e' should not be shadowed by other 'e'
assert config.b.m.e == 4000 # shadowed by untyped but casted to type
with pytest.raises(AttributeError) as exc_info:
config.b.d.e
assert "no attribute 'e'" in str(exc_info.value)
# config.b.d overrides a dict with a value
with pytest.raises(ValueError) as exc_info:
config.b.dump()
assert "conflicts" in str(exc_info.value)
def test_modified_singlesource_subsection(self):
self.globalconf = LayeredConfig(Defaults({
'download_text': None,
'base': {}
}),
cascade=True)
# this should't raise an AttributeError
self.globalconf.base.download_text
# this shouldn't, either
self.globalconf.base.download_text = "WHAT"
def test_set_novalue(self):
# it should be possible to set values that are defined in any
# of the configsources, even though only typing information
# exists there.
cfg = LayeredConfig(Defaults({'placeholder': int}), Commandline([]))
cfg.placeholder = 42
# but it shouldn't be possible to set values that hasn't been
# defined anywhere.
with self.assertRaises(AttributeError):
cfg.nonexistent = 43
def test_get(self):
cfg = LayeredConfig(
Defaults({
'codedefaults': 'yes',
'force': False,
'home': '/usr/home'
}), INIFile('simple.ini'))
# and then do a bunch of get() calls with optional fallbacks
self.assertEqual("yes", LayeredConfig.get(cfg, "codedefaults"))
self.assertEqual("mydata", LayeredConfig.get(cfg, "home"))
self.assertEqual(None, LayeredConfig.get(cfg, "nonexistent"))
self.assertEqual("NO!", LayeredConfig.get(cfg, "nonexistent", "NO!"))
def test_typed_commandline(self):
cmdline = [
'--home=mydata', '--processes=4', '--force=True', '--extra=foo',
'--extra=bar', '--implicitboolean', '--mymodule-force=False',
'--mymodule-extra=foo', '--mymodule-extra=baz',
'--mymodule-expires=2014-10-15',
'--mymodule-arbitrary-nesting-depth=works', '--extramodule-unique'
]
cfg = LayeredConfig(Defaults(self.types), Commandline(cmdline))
self._test_config_subsections(cfg)
self.assertTrue(cfg.implicitboolean)
self.assertIs(type(cfg.implicitboolean), bool)
def test_set(self):
# a value is set in a particular underlying source, and the
# dirty flag isn't set.
cfg = LayeredConfig(INIFile("simple.ini"))
LayeredConfig.set(cfg, 'expires', date(2013, 9, 18), "inifile")
# NOTE: For this config, where no type information is
# available for key 'expires', INIFile.set will convert the
# date object to a string, at which point typing is lost.
# Therefore this commmented-out test will fail
# self.assertEqual(date(2013, 9, 18), cfg.expires)
self.assertEqual("2013-09-18", cfg.expires)
self.assertFalse(cfg._sources[0].dirty)
def test_layered_setdefault():
source1 = DictSource({'a': 1, 'b': {'c': 2}})
source2 = DictSource({'x': 6, 'b': {'y': 7}})
config = LayeredConfig(source1, source2)
assert config.setdefault('a', 10) == 1
assert config.setdefault('nonexisting', 10) == 10
assert config.nonexisting == 10
assert 'nonexisting' in source2
assert config.b.setdefault('nonexisting', 20) == 20
assert config.b.nonexisting == 20
assert 'nonexisting' in source2.b
def test_construct(self):
uri = "http://example.org/doc"
with open("testconstructtemplate.rq", "wb") as fp:
fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/>
CONSTRUCT { ?s ?p ?o . }
WHERE { ?s ?p ?o .
<%(uri)s> ?p ?o . }
""".encode())
g = Graph()
g.bind("dcterms", str(DCTERMS))
g.add((URIRef(uri), DCTERMS.title, Literal("Document title")))
config = {
'connect.return_value': Mock(**{'construct.return_value': g})
}
printmock = MagicMock()
with patch('ferenda.devel.TripleStore', **config):
with patch('builtins.print', printmock):
d = Devel()
d.config = LayeredConfig(
Defaults({
'storetype': 'a',
'storelocation': 'b',
'storerepository': 'c'
}))
d.construct("testconstructtemplate.rq", uri)
want = """
# Constructing the following from b, repository c, type a
# PREFIX dcterms: <http://purl.org/dc/terms/>
#
# CONSTRUCT { ?s ?p ?o . }
# WHERE { ?s ?p ?o .
# <http://example.org/doc> ?p ?o . }
#
@prefix dcterms: <http://purl.org/dc/terms/> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
<http://example.org/doc> dcterms:title "Document title" .
# 1 triples constructed in 0.001s
""".strip()
got = "\n".join([x[1][0] for x in printmock.mock_calls])
self.maxDiff = None
self.assertEqual(self.mask_time(want), self.mask_time(got))
os.unlink("testconstructtemplate.rq")
def test_layered_dump():
config = LayeredConfig(DictSource({
'a': 1,
'b': {
'c': 2
}
}), DictSource({'a': '10'}), DictSource({
'x': 6,
'b': {
'y': 7
}
}))
assert config.dump() == {'a': '10', 'b': {'c': 2, 'y': 7}, 'x': 6}
def test_typed_novalue(self):
# this cmdline only sets some of the settings. The test is
# that the rest should raise AttributeError (not return None,
# as was the previous behaviour), and that __iter__ should not
# include them.
cmdline = ['--processes=4', '--force=False']
cfg = LayeredConfig(Defaults(self.types), Commandline(cmdline))
self.assertEqual(4, cfg.processes)
self.assertIsInstance(cfg.processes, int)
with self.assertRaises(AttributeError):
cfg.home
with self.assertRaises(AttributeError):
cfg.extra
self.assertEqual(set(['processes', 'force']), set(list(cfg)))
def test_layered_config_with_untyped_source():
typed_source1 = {'x': 5, 'b': {'y': 6}}
typed_source2 = {'a': 1, 'b': {'c': 2}}
untyped_source1 = io.StringIO(
pytest.helpers.unindent(u"""
[__root__]
a=11
"""))
untyped_source2 = io.StringIO(
pytest.helpers.unindent(u"""
[__root__]
a=10
x=50
[b]
c=20
y=60
[b.d]
e=30
"""))
typed1 = DictSource(typed_source1)
typed2 = DictSource(typed_source2)
untyped1 = INIFile(untyped_source1)
untyped2 = INIFile(untyped_source2, subsection_token='.')
config = LayeredConfig(typed1, typed2, untyped1, untyped2)
assert typed1.x == 5
assert typed1.b.y == 6
assert typed2.a == 1
assert typed2.b.c == 2
with pytest.raises(KeyError):
typed2.b.d.e
assert untyped1.a == '11'
assert untyped2.a == '10'
assert untyped2.b.c == '20'
assert untyped2.b.d.e == '30'
assert config.a == 10 # found in first typed source
assert config.x == 50 # found in second typed source
assert config.b.c == 20
assert config.b.y == 60
assert config.b.d.e == '30'
