def test_write(self):
cfg = LayeredConfig(INIFile("complex.ini"))
cfg.mymodule.expires = date(2014, 10, 24)
cfg.mymodule.extra = ['foo', 'baz', 'quux']
# calling write for any submodule will force a write of the
# entire config file
LayeredConfig.write(cfg.mymodule)
want = """[__root__]
home = mydata
processes = 4
force = True
extra = foo, bar
[mymodule]
force = False
extra = foo, baz, quux
expires = 2014-10-24
[extramodule]
unique = True
"""
with open("complex.ini") as fp:
got = fp.read().replace("\r\n", "\n")
self.assertEqual(want, got)
def test_inifile_nonexistent(self):
logging.getLogger().setLevel(logging.CRITICAL)
cfg = LayeredConfig(INIFile("nonexistent.ini"))
self.assertEqual([], list(cfg))
# make sure a nonexistent inifile doesn't interfere with the
# rest of the LayeredConfig object
defobj = Defaults({'datadir': 'something'})
iniobj = INIFile("nonexistent.ini")
cfg = LayeredConfig(defobj, iniobj)
self.assertEqual("something", cfg.datadir)
# and make sure it's settable (should set up the INIFile
# object and affect it, and leave the defaults dict untouched
# as it's the lowest priority)
cfg.datadir = "else"
self.assertEqual("else", cfg.datadir)
self.assertEqual("else", iniobj.get("datadir"))
self.assertEqual("something", defobj.get("datadir"))
# same as above, but with a "empty" INIFile object
iniobj = INIFile()
cfg = LayeredConfig(defobj, iniobj)
self.assertEqual("something", cfg.datadir)
cfg.datadir = "else"
self.assertEqual("else", cfg.datadir)
def test_write(self):
self.maxDiff = None
cfg = LayeredConfig(self.complex)
cfg.mymodule.expires = date(2014, 10, 24)
# calling write for any submodule will force a write of the
# entire config file
LayeredConfig.write(cfg.mymodule)
want = """{
"extra": [
"foo",
"bar"
],
"extramodule": {
"unique": true
},
"force": true,
"home": "mydata",
"mymodule": {
"arbitrary": {
"nesting": {
"depth": "works"
}
},
"expires": "2014-10-24",
"extra": [
"foo",
"baz"
],
"force": false
},
"processes": 4
}"""
with open("complex.json") as fp:
got = fp.read().replace("\r\n", "\n")
self.assertEqual(want, got)
def test_write(self):
cfg = LayeredConfig(self.complex)
cfg.mymodule.expires = date(2014, 10, 24)
# calling write for any submodule will force a write of the
# entire config file
LayeredConfig.write(cfg.mymodule)
# note that pyyaml sorts keys alphabetically and has specific
# ideas on how to format the result (controllable through
# mostly-undocumented args to dump())
want = """
extra:
- foo
- bar
extramodule:
unique: true
force: true
home: mydata
mymodule:
arbitrary:
nesting:
depth: works
expires: 2014-10-24
extra:
- foo
- baz
force: false
processes: 4
""".lstrip()
with open("complex.yaml") as fp:
got = fp.read().replace("\r\n", "\n")
self.assertEqual(want, got)
def download(self, basefile=None):
if self.config.ipbasedurls:
self._make_ipbasedurls()
urlmap_path = self.store.path("urls", "downloaded", ".map",
storage_policy="file")
self.urlmap = {}
if os.path.exists(urlmap_path):
with codecs.open(urlmap_path, encoding="utf-8") as fp:
for line in fp:
url, attachment = line.split("\t")
self.urlmap[url] = attachment.strip()
if basefile:
return super(PropTrips, self).download(basefile)
try:
now = datetime.now()
if ('lastyear' in self.config and
self.config.lastyear and
not self.config.refresh):
maxyear = "%s/%s" % (now.year, (now.year + 1) % 100)
while self.config.lastyear != maxyear:
r = self.inner_download()
else:
self.config.lastyear = ''
r = self.inner_download()
self.config.lastyear = "%s/%s" % (now.year - 1,
(now.year % 100))
LayeredConfig.write(self.config) # assume we have data to write
return r
finally:
with codecs.open(urlmap_path, "w", encoding="utf-8") as fp:
for url, attachment in self.urlmap.items():
fp.write("%s\t%s\n" % (url, attachment))
def _make_files(self, option, filedir, combinefile=None, combinefunc=None):
urls = []
buf = BytesIO()
processed = set()
# eg. self.config.cssfiles
if getattr(self.config, option): # it's possible to set eg
# cssfiles=None when
# creating the Resources
# object
for f in getattr(self.config, option):
urls.append(self._process_file(f, buf, filedir, "ferenda.ini"))
processed.add(f)
for repo in self.repos:
# FIXME: create a more generic way of optionally
# signalling to a repo that "Hey, now it's time to create
# your resources if you can"
if repo.__class__.__name__ == "SFS" and option == "imgfiles":
self.log.info("calling into SFS._makeimages()")
LayeredConfig.set(repo.config, 'imgfiles', repo._makeimages())
for f in getattr(repo.config, option):
if f in processed:
continue
urls.append(self._process_file(f, buf, filedir, repo.alias))
processed.add(f)
urls = list(filter(None, urls))
if combinefile:
txt = buf.getvalue().decode('utf-8')
util.writefile(combinefile, combinefunc(txt))
return [self._filepath_to_urlpath(combinefile, 2)]
else:
return urls
def test_write(self):
self.maxDiff = None
cfg = LayeredConfig(self.complex)
cfg.mymodule.expires = date(2014, 10, 24)
cfg.mymodule.extra.append('quux')
# calling write for any submodule will force a write of the
# entire config file
LayeredConfig.write(cfg.mymodule)
# note: plistlib creates files with tabs, not spaces.
want = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>extra</key>
<array>
<string>foo</string>
<string>bar</string>
</array>
<key>extramodule</key>
<dict>
<key>unique</key>
<true/>
</dict>
<key>force</key>
<true/>
<key>home</key>
<string>mydata</string>
<key>mymodule</key>
<dict>
<key>arbitrary</key>
<dict>
<key>nesting</key>
<dict>
<key>depth</key>
<string>works</string>
</dict>
</dict>
<key>expires</key>
<string>2014-10-24</string>
<key>extra</key>
<array>
<string>foo</string>
<string>baz</string>
<string>quux</string>
</array>
<key>force</key>
<false/>
</dict>
<key>processes</key>
<integer>4</integer>
</dict>
</plist>
"""
if sys.version_info < (2,7,0): # pragma: no cover
# on py26, the doctype includes "Apple Computer" not "Apple"...
want = want.replace("//Apple//", "//Apple Computer//")
with open("complex.plist") as fp:
got = fp.read().replace("\r\n", "\n")
self.assertEqual(want, got)
def test_get(self):
cfg = LayeredConfig(Defaults({'codedefaults': 'yes',
'force': False,
'home': '/usr/home'}),
INIFile('simple.ini'))
# and then do a bunch of get() calls with optional fallbacks
self.assertEqual("yes", LayeredConfig.get(cfg, "codedefaults"))
self.assertEqual("mydata", LayeredConfig.get(cfg, "home"))
self.assertEqual(None, LayeredConfig.get(cfg, "nonexistent"))
self.assertEqual("NO!", LayeredConfig.get(cfg, "nonexistent", "NO!"))
def test_set_novalue(self):
# it should be possible to set values that are defined in any
# of the configsources, even though only typing information
# exists there.
cfg = LayeredConfig(Defaults({'placeholder': int}),
Commandline([]))
cfg.placeholder = 42
# but it shouldn't be possible to set values that hasn't been
# defined anywhere.
with self.assertRaises(AttributeError):
cfg.nonexistent = 43
def test_set(self):
# a value is set in a particular underlying source, and the
# dirty flag isn't set.
cfg = LayeredConfig(INIFile("simple.ini"))
LayeredConfig.set(cfg, 'expires', date(2013, 9, 18),
"inifile")
# NOTE: For this config, where no type information is
# available for key 'expires', INIFile.set will convert the
# date object to a string, at which point typing is lost.
# Therefore this commmented-out test will fail
# self.assertEqual(date(2013, 9, 18), cfg.expires)
self.assertEqual("2013-09-18", cfg.expires)
self.assertFalse(cfg._sources[0].dirty)
def test_typed_override(self):
# make sure this auto-typing isn't run for bools
types = {'logfile': True}
cmdline = ["--logfile=out.log"]
cfg = LayeredConfig(Defaults(types), Commandline(cmdline))
self.assertEqual(cfg.logfile, "out.log")
def test_write(self):
def indexfilter(node):
# remove keys like createdIdex / modifiedIndex whose
# values always changes
if isinstance(node, dict):
for key in list(node.keys()):
if key == "nodes":
indexfilter(node[key])
elif key.endswith("Index"):
del node[key]
else:
node[:] = sorted(node, key=itemgetter('key'))
for subnode in node:
indexfilter(subnode)
cfg = LayeredConfig(self.complex)
cfg.mymodule.expires = date(2014, 10, 24)
LayeredConfig.write(cfg.mymodule)
want = """
{
"dir": true,
"key": "/",
"nodes": [
{
"createdIndex": 4627,
"key": "/home",
"modifiedIndex": 4627,
"value": "mydata"
},
{
"createdIndex": 4628,
"key": "/processes",
"modifiedIndex": 4628,
"value": "4"
},
{
"createdIndex": 4629,
"key": "/force",
"modifiedIndex": 4629,
"value": "true"
},
{
"createdIndex": 4630,
"key": "/extra",
"modifiedIndex": 4630,
"value": "foo, bar"
},
{
"createdIndex": 4631,
"dir": true,
"key": "/mymodule",
"modifiedIndex": 4631,
"nodes": [
{
"createdIndex": 4631,
"key": "/mymodule/force",
"modifiedIndex": 4631,
"value": "false"
},
{
"createdIndex": 4632,
"key": "/mymodule/extra",
"modifiedIndex": 4632,
"value": "foo, baz"
},
{
"createdIndex": 4633,
"key": "/mymodule/expires",
"modifiedIndex": 4633,
"value": "2014-10-24"
},
{
"createdIndex": 4634,
"dir": true,
"key": "/mymodule/arbitrary",
"modifiedIndex": 4634,
"nodes": [
{
"createdIndex": 4634,
"dir": true,
"key": "/mymodule/arbitrary/nesting",
"modifiedIndex": 4634,
"nodes": [
{
"createdIndex": 4634,
"key": "/mymodule/arbitrary/nesting/depth",
"modifiedIndex": 4634,
"value": "works"
}
]
}
]
}
]
},
{
"createdIndex": 4635,
"dir": true,
"key": "/extramodule",
"modifiedIndex": 4635,
"nodes": [
{
"createdIndex": 4635,
"key": "/extramodule/unique",
"modifiedIndex": 4635,
"value": "true"
}
]
}
]
}"""
want = json.loads(want)
indexfilter(want)
got = requests.get("http://localhost:4001/v2/keys/?recursive=true").json()['node']
indexfilter(got)
self.assertEqual(want, got)
myinifile = INIFile("myapp.ini")
# end inifile
# begin environment
env = {'MYAPP_HOME': 'C:\\Progra~1\\MyApp', 'MYAPP_SUBMODULE_RETRY': 'True'}
myenv = Environment(env, prefix="MYAPP_")
# end environment
# begin commandline
mycmdline = Commandline(['-f', '--home=/opt/myapp', '--times=2', '--dostuff'])
rest = mycmdline.rest
# end commandline
assert rest == ['-f']
# begin makeconfig
cfg = LayeredConfig(mydefaults, myinifile, myenv, mycmdline)
# end makeconfig
import os
def do_stuff(action, idx):
pass
# begin useconfig
print("%s starting, home in %s" % (cfg.name, cfg.home))
# end useconfig
# begin usetyping
delay = date.today() - cfg.duedate # date
if cfg.dostuff: # bool
items,
'doccount':
len(list(self.store.list_basefiles_for("_postgenerate")))
})
# end frontpage_content
from ferenda import manager
from layeredconfig import LayeredConfig, Defaults
import sys
manager.setup_logger("DEBUG")
d = RFCs(downloadmax=5)
d.download()
for basefile in d.store.list_basefiles_for("parse"):
d.parse(basefile)
RFCs.setup("relate", LayeredConfig(Defaults(d.get_default_options())))
for basefile in d.store.list_basefiles_for("relate"):
d.relate(basefile)
RFCs.teardown("relate", LayeredConfig(Defaults(d.get_default_options())))
manager.makeresources([d])
for basefile in d.store.list_basefiles_for("generate"):
d.generate(basefile)
d.toc()
d.news()
manager.frontpage([d])
shutil.rmtree("data")
return_value = True
# begin example
from layeredconfig import LayeredConfig, PyFile, Defaults
from datetime import date, datetime
conf = LayeredConfig(
Defaults({
'home': '/tmp/myapp',
'name': 'MyApp',
'dostuff': False,
'times': 4,
'duedate': date(2014, 10, 30),
'things': ['Huey', 'Dewey', 'Louie'],
'submodule': {
'retry': False,
'lastrun': datetime(2014, 10, 30, 16, 40, 22)
}
}), PyFile("conf.py"))
# end example
from datetime import date, datetime
import os
assert conf.home == os.getcwd()
assert conf.name == 'My App'
assert conf.dostuff is True
assert conf.times == 4
assert conf.duedate == date.today()
assert conf.things == ['Huey', 'Dewey', 'Louie']
assert conf.submodule.lastrun == datetime(2014, 10, 30, 16, 40, 22)
assert conf.submodule.retry is True
return_value = conf.name
def test_write_noconfigfile(self):
cfg = LayeredConfig(Defaults({'lastrun':
datetime(2012, 9, 18, 15, 41, 0)}))
cfg.lastrun = datetime(2013, 9, 18, 15, 41, 0)
LayeredConfig.write(cfg)
from layeredconfig import LayeredConfig, PyFile
# begin example
conf = LayeredConfig(PyFile("defaults.py"), PyFile("conf.py"))
# end example
from datetime import date, datetime
import os
assert conf.home == os.getcwd()
assert conf.name == 'My App'
assert conf.dostuff is True
assert conf.times == 4
assert conf.duedate == date.today()
assert conf.things == ['Huey', 'Dewey', 'Louie']
assert conf.submodule.lastrun == datetime(2014, 10, 30, 16, 40, 22)
assert conf.submodule.retry is True
return_value = conf.name
def test_typed_inifile(self):
cfg = LayeredConfig(Defaults(self.types), INIFile("complex.ini"))
self.supported_types = (str, bool, int, list, date, datetime)
self.supports_nesting = False
self._test_config_subsections(cfg)
delay = date.today() - cfg.duedate # date
if cfg.dostuff: # bool
for i in range(cfg.times): # int
print(", ".join(cfg.things)) # list
# end usetyping
# begin usesubconfig
subcfg = cfg.submodule
if subcfg.retry:
print(subcfg.lastrun.isoformat())
# end usesubconfig
try:
print(subcfg.home)
except AttributeError:
pass
# begin usecascade
cfg = LayeredConfig(mydefaults, myinifile, myenv, mycmdline, cascade=True)
subcfg = cfg.submodule
print(subcfg.home) # prints '/opt/myapp', from Commandline source root section
# end usecascade
assert subcfg.home == '/opt/myapp'
# begin writeconfig
subcfg.lastrun = datetime.now()
LayeredConfig.write(cfg)
# end writeconfig
return_value = True
inifile = INIFile("myapp.ini")
# end inifile
# begin argparse
parser = argparse.ArgumentParser("This is a simple program")
parser.add_argument("--home", help="The home directory of the app")
parser.add_argument('--dostuff', action="store_true", help="Do some work")
parser.add_argument("-t", "--times", type=int, help="Number of times to do it")
parser.add_argument('--things', action="append", help="Extra things to crunch")
parser.add_argument('--retry', action="store_true", help="Try again")
parser.add_argument("file", metavar="FILE", help="The filename to process")
# end argparse
# begin layeredconfig
sys.argv = ['./myapp.py', '--home=/opt/myapp', '-t=2', '--dostuff', 'file.txt']
cfg = LayeredConfig(defaults, inifile, Commandline(parser=parser))
print("Starting %s in %s for %r times (doing work: %s)" %
(cfg.name, cfg.home, cfg.times, cfg.dostuff))
# should print "Starting MyApp in /opt/myapp for 2 times (doing work: True)"
# end layeredconfig
# begin showhelp
sys.argv = ['./myapp.py', '-h']
cfg = LayeredConfig(defaults, inifile, Commandline(parser=parser))
# end showhelp
# begin nodefaults
# NOTE: we never reach this because the previous call to -h will have
# called sys.exit
def test_persistant_subrepo_config(self):
self.repo.custom("set", "blahonga")
LayeredConfig.write(self.repo.config)
self.repo = self._createrepo()
self.assertEqual("SubrepoBSubclass: blahonga", self.repo.custom("get"))
def test_modified(self):
defaults = {'lastdownload': None}
cfg = LayeredConfig(Defaults(defaults))
now = datetime.now()
cfg.lastdownload = now
self.assertEqual(cfg.lastdownload, now)
def test_newint(self):
os.environ['FERENDA_DOWNLOADMAX'] = '3'
config = LayeredConfig(Defaults({'downloadmax': int}),
Environment(prefix="FERENDA_"))
self.assertEqual(3, config.downloadmax)
self.assertIsInstance(config.downloadmax, int)