def test_inifile_nonexistent(self):
logging.getLogger().setLevel(logging.CRITICAL)
cfg = LayeredConfig(INIFile("nonexistent.ini"))
self.assertEqual([], list(cfg))
# make sure a nonexistent inifile doesn't interfere with the
# rest of the LayeredConfig object
defobj = Defaults({'datadir': 'something'})
iniobj = INIFile("nonexistent.ini")
cfg = LayeredConfig(defobj, iniobj)
self.assertEqual("something", cfg.datadir)
# and make sure it's settable (should set up the INIFile
# object and affect it, and leave the defaults dict untouched
# as it's the lowest priority)
cfg.datadir = "else"
self.assertEqual("else", cfg.datadir)
self.assertEqual("else", iniobj.get("datadir"))
self.assertEqual("something", defobj.get("datadir"))
# same as above, but with a "empty" INIFile object
iniobj = INIFile()
cfg = LayeredConfig(defobj, iniobj)
self.assertEqual("something", cfg.datadir)
cfg.datadir = "else"
self.assertEqual("else", cfg.datadir)
def setUp(self):
super(WSGI,self).setUp()
if self.storelocation.startswith("data/"):
self.storelocation = self.storelocation.replace("data", self.datadir)
if self.indexlocation.startswith("data/"):
self.indexlocation = self.indexlocation.replace("data", self.datadir)
self.put_files_in_place()
# use self.repo (simple testcases) or self.repos (complex
# testcases like AdvancedAPI)?
if hasattr(self, 'repos'):
repos = self.repos
else:
repos = [self.repo]
# print("making app: %s %s" % (self.storetype, self.indextype))
config = LayeredConfig(Defaults({'datadir': self.datadir,
'apiendpoint': '/myapi/',
'searchendpoint': '/mysearch/',
'url': 'http://localhost:8000/',
'storetype': self.storetype,
'storelocation': self.storelocation,
'storerepository': self.storerepository,
'indextype': self.indextype,
'indexlocation': self.indexlocation,
'wsgiappclass': 'ferenda.WSGIApp',
'legacyapi': False,
'wsgiexceptionhandler': True}))
self.app = manager.make_wsgi_app(config, repos=repos)
self.builder = EnvironBuilder('/', base_url="http://localhost:8000/",
headers={"Accept": DEFAULT_HTTP_ACCEPT})
def test_modified_subsections(self):
defaults = {'force': False, 'home': 'thisdata', 'loglevel': 'INFO'}
cmdline = ['--mymodule-home=thatdata', '--mymodule-force']
cfg = LayeredConfig(Defaults(defaults),
INIFile("complex.ini"),
Commandline(cmdline),
cascade=True)
cfg.mymodule.expires = date(2014, 10, 24)
def test_select(self):
uri = "http://example.org/doc"
with open("testselecttemplate.rq", "wb") as fp:
fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/>
SELECT ?p ?o
WHERE { <%(uri)s> ?p ?o . }
""".encode())
result = """
[
{
"p": "http://purl.org/dc/terms/title",
"o": "Document title"
},
{
"p": "http://purl.org/dc/terms/identifier",
"o": "Document ID"
}
]""".lstrip().encode("utf-8")
config = {
'connect.return_value': Mock(**{'select.return_value': result})
}
printmock = MagicMock()
with patch('ferenda.devel.TripleStore', **config):
with patch('builtins.print', printmock):
d = Devel()
d.config = LayeredConfig(
Defaults({
'storetype': 'a',
'storelocation': 'b',
'storerepository': 'c'
}))
d.select("testselecttemplate.rq", uri)
want = """
# Constructing the following from b, repository c, type a
# PREFIX dcterms: <http://purl.org/dc/terms/>
#
# SELECT ?p ?o
# WHERE { <http://example.org/doc> ?p ?o . }
#
[
{
"p": "http://purl.org/dc/terms/title",
"o": "Document title"
},
{
"p": "http://purl.org/dc/terms/identifier",
"o": "Document ID"
}
]
# Selected in 0.001s
""".strip()
got = "\n".join([x[1][0] for x in printmock.mock_calls])
self.maxDiff = None
self.assertEqual(self.mask_time(want), self.mask_time(got))
os.unlink("testselecttemplate.rq")
def test_layered(self):
defaults = {'home': 'someplace'}
cmdline = ['--home=anotherplace']
env = {'MYAPP_HOME': 'yourdata'}
cfg = LayeredConfig(Defaults(defaults))
self.assertEqual(cfg.home, 'someplace')
cfg = LayeredConfig(Defaults(defaults), INIFile("simple.ini"))
self.assertEqual(cfg.home, 'mydata')
cfg = LayeredConfig(Defaults(defaults), INIFile("simple.ini"),
Environment(env, prefix="MYAPP_"))
self.assertEqual(cfg.home, 'yourdata')
cfg = LayeredConfig(Defaults(defaults), INIFile("simple.ini"),
Environment(env, prefix="MYAPP_"),
Commandline(cmdline))
self.assertEqual(cfg.home, 'anotherplace')
self.assertEqual(
['home', 'processes', 'force', 'extra', 'expires', 'lastrun'],
list(cfg))
def test_modified_singlesource_subsection(self):
self.globalconf = LayeredConfig(Defaults({
'download_text': None,
'base': {}
}),
cascade=True)
# this should't raise an AttributeError
self.globalconf.base.download_text
# this shouldn't, either
self.globalconf.base.download_text = "WHAT"
def test_set_novalue(self):
# it should be possible to set values that are defined in any
# of the configsources, even though only typing information
# exists there.
cfg = LayeredConfig(Defaults({'placeholder': int}), Commandline([]))
cfg.placeholder = 42
# but it shouldn't be possible to set values that hasn't been
# defined anywhere.
with self.assertRaises(AttributeError):
cfg.nonexistent = 43
def test_typed_commandline(self):
cmdline = [
'--home=mydata', '--processes=4', '--force=True', '--extra=foo',
'--extra=bar', '--implicitboolean', '--mymodule-force=False',
'--mymodule-extra=foo', '--mymodule-extra=baz',
'--mymodule-expires=2014-10-15',
'--mymodule-arbitrary-nesting-depth=works', '--extramodule-unique'
]
cfg = LayeredConfig(Defaults(self.types), Commandline(cmdline))
self._test_config_subsections(cfg)
self.assertTrue(cfg.implicitboolean)
self.assertIs(type(cfg.implicitboolean), bool)
def test_get(self):
cfg = LayeredConfig(
Defaults({
'codedefaults': 'yes',
'force': False,
'home': '/usr/home'
}), INIFile('simple.ini'))
# and then do a bunch of get() calls with optional fallbacks
self.assertEqual("yes", LayeredConfig.get(cfg, "codedefaults"))
self.assertEqual("mydata", LayeredConfig.get(cfg, "home"))
self.assertEqual(None, LayeredConfig.get(cfg, "nonexistent"))
self.assertEqual("NO!", LayeredConfig.get(cfg, "nonexistent", "NO!"))
def test_typed_novalue(self):
# this cmdline only sets some of the settings. The test is
# that the rest should raise AttributeError (not return None,
# as was the previous behaviour), and that __iter__ should not
# include them.
cmdline = ['--processes=4', '--force=False']
cfg = LayeredConfig(Defaults(self.types), Commandline(cmdline))
self.assertEqual(4, cfg.processes)
self.assertIsInstance(cfg.processes, int)
with self.assertRaises(AttributeError):
cfg.home
with self.assertRaises(AttributeError):
cfg.extra
self.assertEqual(set(['processes', 'force']), set(list(cfg)))
def test_construct(self):
uri = "http://example.org/doc"
with open("testconstructtemplate.rq", "wb") as fp:
fp.write("""PREFIX dcterms: <http://purl.org/dc/terms/>
CONSTRUCT { ?s ?p ?o . }
WHERE { ?s ?p ?o .
<%(uri)s> ?p ?o . }
""".encode())
g = Graph()
g.bind("dcterms", str(DCTERMS))
g.add((URIRef(uri), DCTERMS.title, Literal("Document title")))
config = {
'connect.return_value': Mock(**{'construct.return_value': g})
}
printmock = MagicMock()
with patch('ferenda.devel.TripleStore', **config):
with patch('builtins.print', printmock):
d = Devel()
d.config = LayeredConfig(
Defaults({
'storetype': 'a',
'storelocation': 'b',
'storerepository': 'c'
}))
d.construct("testconstructtemplate.rq", uri)
want = """
# Constructing the following from b, repository c, type a
# PREFIX dcterms: <http://purl.org/dc/terms/>
#
# CONSTRUCT { ?s ?p ?o . }
# WHERE { ?s ?p ?o .
# <http://example.org/doc> ?p ?o . }
#
@prefix dcterms: <http://purl.org/dc/terms/> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
@prefix xml: <http://www.w3.org/XML/1998/namespace> .
@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
<http://example.org/doc> dcterms:title "Document title" .
# 1 triples constructed in 0.001s
""".strip()
got = "\n".join([x[1][0] for x in printmock.mock_calls])
self.maxDiff = None
self.assertEqual(self.mask_time(want), self.mask_time(got))
os.unlink("testconstructtemplate.rq")
def test_typed_commandline_cascade(self):
# the test here is that __getattribute__ must determine that
# subconfig.force is not typed in itself, and fetch type
# information from the root of defaults
defaults = {'force': True, 'lastdownload': datetime, 'mymodule': {}}
cmdline = ['--mymodule-force=False']
cfg = LayeredConfig(Defaults(defaults),
Commandline(cmdline),
cascade=True)
subconfig = getattr(cfg, 'mymodule')
self.assertIs(type(subconfig.force), bool)
self.assertEqual(subconfig.force, False)
# test typed config values that have no actual value. Since
# they have no value, they should raise AtttributeError
with self.assertRaises(AttributeError):
self.assertEqual(cfg.lastdownload, None)
with self.assertRaises(AttributeError):
self.assertEqual(subconfig.lastdownload, None)
def load_config():
"""
Searches a standard set of locations for .farmer.yml, and parses the first
match.
"""
pwd = os.getcwd()
paths = [
os.path.join(pwd, '.farmer.yml'),
os.path.join(pwd, '.farmer', 'farmer.yml'),
os.path.join(DEFAULT_CONFIG_DIR, 'farmer.yml')
]
config_file = None
for path in paths:
if os.path.exists(path):
config_file = path
break
return LayeredConfig(
Defaults(DEFAULTS),
RoundTripYAMLFile(config_file),
Environment(prefix='FARMER_', sectionsep='__'),
)
def __init__(self, repos, resourcedir, **kwargs):
# FIXME: document what kwargs could be (particularly 'combineresources')
self.repos = repos
self.resourcedir = resourcedir
defaults = DocumentRepository.get_default_options()
defaults.update(kwargs)
self.config = LayeredConfig(Defaults(defaults))
# the below call to setup_logger alters the logging level of
# the root logger, which can't be good practice. Also, we
# should probably not log to the root logger, but rather to
# ferenda.resources.
#
# from ferenda.manager import setup_logger
# self.log = setup_logger()
self.log = logging.getLogger("ferenda.resources")
# FIXME: How should we set up a global loadpath from the
# individual repos?
loadpaths = [ResourceLoader.make_loadpath(repo) for repo in repos]
loadpath = ["."] # cwd always has priority -- makes sense?
for subpath in loadpaths:
for p in subpath:
if p not in loadpath:
loadpath.append(p)
self.resourceloader = ResourceLoader(*loadpath)
def setUp(self):
self.datadir = tempfile.mkdtemp()
self.basefile = "1"
self.store = DocumentStore(self.datadir + "/base")
self.d = Devel()
self.globalconf = LayeredConfig(Defaults({
'datadir': self.datadir,
'patchdir': self.datadir,
'download_text': None,
'intermediate_text': None,
'devel': {
'class': 'ferenda.Devel'
},
'base': {
'class': 'testDevel.MockRepo'
},
'koi8': {
'class': 'testDevel.Koi8Repo'
}
}),
cascade=True)
self.d.config = self.globalconf.devel
self.d.config.download_text
self.d.config.download_text = "what"
def test_typed_override(self):
# make sure this auto-typing isn't run for bools
types = {'logfile': True}
cmdline = ["--logfile=out.log"]
cfg = LayeredConfig(Defaults(types), Commandline(cmdline))
self.assertEqual(cfg.logfile, "out.log")
# begin example
from layeredconfig import LayeredConfig, PyFile, Defaults
from datetime import date, datetime
conf = LayeredConfig(
Defaults({
'home': '/tmp/myapp',
'name': 'MyApp',
'dostuff': False,
'times': 4,
'duedate': date(2014, 10, 30),
'things': ['Huey', 'Dewey', 'Louie'],
'submodule': {
'retry': False,
'lastrun': datetime(2014, 10, 30, 16, 40, 22)
}
}), PyFile("conf.py"))
# end example
from datetime import date, datetime
import os
assert conf.home == os.getcwd()
assert conf.name == 'My App'
assert conf.dostuff is True
assert conf.times == 4
assert conf.duedate == date.today()
assert conf.things == ['Huey', 'Dewey', 'Louie']
assert conf.submodule.lastrun == datetime(2014, 10, 30, 16, 40, 22)
assert conf.submodule.retry is True
return_value = conf.name
def test_write_noconfigfile(self):
cfg = LayeredConfig(
Defaults({'lastrun': datetime(2012, 9, 18, 15, 41, 0)}))
cfg.lastrun = datetime(2013, 9, 18, 15, 41, 0)
LayeredConfig.write(cfg)
from __future__ import print_function
# begin import
import sys
import argparse
from datetime import date, datetime
from layeredconfig import LayeredConfig, Defaults, INIFile, Commandline
# end import
# begin defaults
defaults = Defaults({
'home': str,
'name': 'MyApp',
'dostuff': bool,
'times': int,
'duedate': date,
'things': list,
'submodule': {
'retry': bool,
'lastrun': datetime
}
})
# end defaults
# begin inifile
with open("myapp.ini", "w") as fp:
fp.write("""[__root__]
home = /tmp/myapp
dostuff = False
times = 4
duedate = 2014-10-30
things = Huey, Dewey, Louie
def test_modified(self):
defaults = {'lastdownload': None}
cfg = LayeredConfig(Defaults(defaults))
now = datetime.now()
cfg.lastdownload = now
self.assertEqual(cfg.lastdownload, now)
def test_list(self):
defaults = {'home': 'mydata', 'subsection': {'processes': 4}}
cfg = LayeredConfig(Defaults(defaults), cascade=True)
self.assertEqual(set(['home', 'processes']), set(cfg.subsection))
items,
'doccount':
len(list(self.store.list_basefiles_for("_postgenerate")))
})
# end frontpage_content
from ferenda import manager
from layeredconfig import LayeredConfig, Defaults
import sys
manager.setup_logger("DEBUG")
d = RFCs(downloadmax=5)
d.download()
for basefile in d.store.list_basefiles_for("parse"):
d.parse(basefile)
RFCs.setup("relate", LayeredConfig(Defaults(d.get_default_options())))
for basefile in d.store.list_basefiles_for("relate"):
d.relate(basefile)
RFCs.teardown("relate", LayeredConfig(Defaults(d.get_default_options())))
manager.makeresources([d])
for basefile in d.store.list_basefiles_for("generate"):
d.generate(basefile)
d.toc()
d.news()
manager.frontpage([d])
shutil.rmtree("data")
return_value = True
def test_typed_inifile(self):
cfg = LayeredConfig(Defaults(self.types), INIFile("complex.ini"))
self.supported_types = (str, bool, int, list, date, datetime)
self.supports_nesting = False
self._test_config_subsections(cfg)
# 1. hard-coded defaults
defaults = {"hello": "is it me you're looking for?"}
# 2. INI configuration file
with open("myapp.ini", "w") as fp:
fp.write("""
[__root__]
hello = kitty
""")
# 3. enironment variables
import os
os.environ['MYAPP_HELLO'] = 'goodbye'
# 4.command-line arguments
import sys
sys.argv = ['./myapp.py', '--hello=world']
# Create a config object that gets settings from these four
# sources.
config = LayeredConfig(Defaults(defaults), INIFile("myapp.ini"),
Environment(prefix="MYAPP_"), Commandline())
# Prints "Hello world!", i.e the value provided by command-line
# arguments. Latter sources take precedence over earlier sources.
print("Hello %s!" % config.hello)
# end firststep
return_value = True
def test_newint(self):
os.environ['FERENDA_DOWNLOADMAX'] = '3'
config = LayeredConfig(Defaults({'downloadmax': int}),
Environment(prefix="FERENDA_"))
self.assertEqual(3, config.downloadmax)
self.assertIsInstance(config.downloadmax, int)