def test_write(self): cfg = LayeredConfig(INIFile("complex.ini")) cfg.mymodule.expires = date(2014, 10, 24) cfg.mymodule.extra = ['foo', 'baz', 'quux'] # calling write for any submodule will force a write of the # entire config file LayeredConfig.write(cfg.mymodule) want = """[__root__] home = mydata processes = 4 force = True extra = foo, bar [mymodule] force = False extra = foo, baz, quux expires = 2014-10-24 [extramodule] unique = True """ with open("complex.ini") as fp: got = fp.read().replace("\r\n", "\n") self.assertEqual(want, got)
def test_inifile_nonexistent(self): logging.getLogger().setLevel(logging.CRITICAL) cfg = LayeredConfig(INIFile("nonexistent.ini")) self.assertEqual([], list(cfg)) # make sure a nonexistent inifile doesn't interfere with the # rest of the LayeredConfig object defobj = Defaults({'datadir': 'something'}) iniobj = INIFile("nonexistent.ini") cfg = LayeredConfig(defobj, iniobj) self.assertEqual("something", cfg.datadir) # and make sure it's settable (should set up the INIFile # object and affect it, and leave the defaults dict untouched # as it's the lowest priority) cfg.datadir = "else" self.assertEqual("else", cfg.datadir) self.assertEqual("else", iniobj.get("datadir")) self.assertEqual("something", defobj.get("datadir")) # same as above, but with a "empty" INIFile object iniobj = INIFile() cfg = LayeredConfig(defobj, iniobj) self.assertEqual("something", cfg.datadir) cfg.datadir = "else" self.assertEqual("else", cfg.datadir)
def test_write(self): self.maxDiff = None cfg = LayeredConfig(self.complex) cfg.mymodule.expires = date(2014, 10, 24) # calling write for any submodule will force a write of the # entire config file LayeredConfig.write(cfg.mymodule) want = """{ "extra": [ "foo", "bar" ], "extramodule": { "unique": true }, "force": true, "home": "mydata", "mymodule": { "arbitrary": { "nesting": { "depth": "works" } }, "expires": "2014-10-24", "extra": [ "foo", "baz" ], "force": false }, "processes": 4 }""" with open("complex.json") as fp: got = fp.read().replace("\r\n", "\n") self.assertEqual(want, got)
def test_write(self): cfg = LayeredConfig(self.complex) cfg.mymodule.expires = date(2014, 10, 24) # calling write for any submodule will force a write of the # entire config file LayeredConfig.write(cfg.mymodule) # note that pyyaml sorts keys alphabetically and has specific # ideas on how to format the result (controllable through # mostly-undocumented args to dump()) want = """ extra: - foo - bar extramodule: unique: true force: true home: mydata mymodule: arbitrary: nesting: depth: works expires: 2014-10-24 extra: - foo - baz force: false processes: 4 """.lstrip() with open("complex.yaml") as fp: got = fp.read().replace("\r\n", "\n") self.assertEqual(want, got)
def download(self, basefile=None): if self.config.ipbasedurls: self._make_ipbasedurls() urlmap_path = self.store.path("urls", "downloaded", ".map", storage_policy="file") self.urlmap = {} if os.path.exists(urlmap_path): with codecs.open(urlmap_path, encoding="utf-8") as fp: for line in fp: url, attachment = line.split("\t") self.urlmap[url] = attachment.strip() if basefile: return super(PropTrips, self).download(basefile) try: now = datetime.now() if ('lastyear' in self.config and self.config.lastyear and not self.config.refresh): maxyear = "%s/%s" % (now.year, (now.year + 1) % 100) while self.config.lastyear != maxyear: r = self.inner_download() else: self.config.lastyear = '' r = self.inner_download() self.config.lastyear = "%s/%s" % (now.year - 1, (now.year % 100)) LayeredConfig.write(self.config) # assume we have data to write return r finally: with codecs.open(urlmap_path, "w", encoding="utf-8") as fp: for url, attachment in self.urlmap.items(): fp.write("%s\t%s\n" % (url, attachment))
def _make_files(self, option, filedir, combinefile=None, combinefunc=None): urls = [] buf = BytesIO() processed = set() # eg. self.config.cssfiles if getattr(self.config, option): # it's possible to set eg # cssfiles=None when # creating the Resources # object for f in getattr(self.config, option): urls.append(self._process_file(f, buf, filedir, "ferenda.ini")) processed.add(f) for repo in self.repos: # FIXME: create a more generic way of optionally # signalling to a repo that "Hey, now it's time to create # your resources if you can" if repo.__class__.__name__ == "SFS" and option == "imgfiles": self.log.info("calling into SFS._makeimages()") LayeredConfig.set(repo.config, 'imgfiles', repo._makeimages()) for f in getattr(repo.config, option): if f in processed: continue urls.append(self._process_file(f, buf, filedir, repo.alias)) processed.add(f) urls = list(filter(None, urls)) if combinefile: txt = buf.getvalue().decode('utf-8') util.writefile(combinefile, combinefunc(txt)) return [self._filepath_to_urlpath(combinefile, 2)] else: return urls
def test_write(self): self.maxDiff = None cfg = LayeredConfig(self.complex) cfg.mymodule.expires = date(2014, 10, 24) cfg.mymodule.extra.append('quux') # calling write for any submodule will force a write of the # entire config file LayeredConfig.write(cfg.mymodule) # note: plistlib creates files with tabs, not spaces. want = """<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> <key>extra</key> <array> <string>foo</string> <string>bar</string> </array> <key>extramodule</key> <dict> <key>unique</key> <true/> </dict> <key>force</key> <true/> <key>home</key> <string>mydata</string> <key>mymodule</key> <dict> <key>arbitrary</key> <dict> <key>nesting</key> <dict> <key>depth</key> <string>works</string> </dict> </dict> <key>expires</key> <string>2014-10-24</string> <key>extra</key> <array> <string>foo</string> <string>baz</string> <string>quux</string> </array> <key>force</key> <false/> </dict> <key>processes</key> <integer>4</integer> </dict> </plist> """ if sys.version_info < (2,7,0): # pragma: no cover # on py26, the doctype includes "Apple Computer" not "Apple"... want = want.replace("//Apple//", "//Apple Computer//") with open("complex.plist") as fp: got = fp.read().replace("\r\n", "\n") self.assertEqual(want, got)
def test_get(self): cfg = LayeredConfig(Defaults({'codedefaults': 'yes', 'force': False, 'home': '/usr/home'}), INIFile('simple.ini')) # and then do a bunch of get() calls with optional fallbacks self.assertEqual("yes", LayeredConfig.get(cfg, "codedefaults")) self.assertEqual("mydata", LayeredConfig.get(cfg, "home")) self.assertEqual(None, LayeredConfig.get(cfg, "nonexistent")) self.assertEqual("NO!", LayeredConfig.get(cfg, "nonexistent", "NO!"))
def test_set_novalue(self): # it should be possible to set values that are defined in any # of the configsources, even though only typing information # exists there. cfg = LayeredConfig(Defaults({'placeholder': int}), Commandline([])) cfg.placeholder = 42 # but it shouldn't be possible to set values that hasn't been # defined anywhere. with self.assertRaises(AttributeError): cfg.nonexistent = 43
def test_set(self): # a value is set in a particular underlying source, and the # dirty flag isn't set. cfg = LayeredConfig(INIFile("simple.ini")) LayeredConfig.set(cfg, 'expires', date(2013, 9, 18), "inifile") # NOTE: For this config, where no type information is # available for key 'expires', INIFile.set will convert the # date object to a string, at which point typing is lost. # Therefore this commmented-out test will fail # self.assertEqual(date(2013, 9, 18), cfg.expires) self.assertEqual("2013-09-18", cfg.expires) self.assertFalse(cfg._sources[0].dirty)
def test_typed_override(self): # make sure this auto-typing isn't run for bools types = {'logfile': True} cmdline = ["--logfile=out.log"] cfg = LayeredConfig(Defaults(types), Commandline(cmdline)) self.assertEqual(cfg.logfile, "out.log")
def test_write(self): def indexfilter(node): # remove keys like createdIdex / modifiedIndex whose # values always changes if isinstance(node, dict): for key in list(node.keys()): if key == "nodes": indexfilter(node[key]) elif key.endswith("Index"): del node[key] else: node[:] = sorted(node, key=itemgetter('key')) for subnode in node: indexfilter(subnode) cfg = LayeredConfig(self.complex) cfg.mymodule.expires = date(2014, 10, 24) LayeredConfig.write(cfg.mymodule) want = """ { "dir": true, "key": "/", "nodes": [ { "createdIndex": 4627, "key": "/home", "modifiedIndex": 4627, "value": "mydata" }, { "createdIndex": 4628, "key": "/processes", "modifiedIndex": 4628, "value": "4" }, { "createdIndex": 4629, "key": "/force", "modifiedIndex": 4629, "value": "true" }, { "createdIndex": 4630, "key": "/extra", "modifiedIndex": 4630, "value": "foo, bar" }, { "createdIndex": 4631, "dir": true, "key": "/mymodule", "modifiedIndex": 4631, "nodes": [ { "createdIndex": 4631, "key": "/mymodule/force", "modifiedIndex": 4631, "value": "false" }, { "createdIndex": 4632, "key": "/mymodule/extra", "modifiedIndex": 4632, "value": "foo, baz" }, { "createdIndex": 4633, "key": "/mymodule/expires", "modifiedIndex": 4633, "value": "2014-10-24" }, { "createdIndex": 4634, "dir": true, "key": "/mymodule/arbitrary", "modifiedIndex": 4634, "nodes": [ { "createdIndex": 4634, "dir": true, "key": "/mymodule/arbitrary/nesting", "modifiedIndex": 4634, "nodes": [ { "createdIndex": 4634, "key": "/mymodule/arbitrary/nesting/depth", "modifiedIndex": 4634, "value": "works" } ] } ] } ] }, { "createdIndex": 4635, "dir": true, "key": "/extramodule", "modifiedIndex": 4635, "nodes": [ { "createdIndex": 4635, "key": "/extramodule/unique", "modifiedIndex": 4635, "value": "true" } ] } ] }""" want = json.loads(want) indexfilter(want) got = requests.get("http://localhost:4001/v2/keys/?recursive=true").json()['node'] indexfilter(got) self.assertEqual(want, got)
myinifile = INIFile("myapp.ini") # end inifile # begin environment env = {'MYAPP_HOME': 'C:\\Progra~1\\MyApp', 'MYAPP_SUBMODULE_RETRY': 'True'} myenv = Environment(env, prefix="MYAPP_") # end environment # begin commandline mycmdline = Commandline(['-f', '--home=/opt/myapp', '--times=2', '--dostuff']) rest = mycmdline.rest # end commandline assert rest == ['-f'] # begin makeconfig cfg = LayeredConfig(mydefaults, myinifile, myenv, mycmdline) # end makeconfig import os def do_stuff(action, idx): pass # begin useconfig print("%s starting, home in %s" % (cfg.name, cfg.home)) # end useconfig # begin usetyping delay = date.today() - cfg.duedate # date if cfg.dostuff: # bool
items, 'doccount': len(list(self.store.list_basefiles_for("_postgenerate"))) }) # end frontpage_content from ferenda import manager from layeredconfig import LayeredConfig, Defaults import sys manager.setup_logger("DEBUG") d = RFCs(downloadmax=5) d.download() for basefile in d.store.list_basefiles_for("parse"): d.parse(basefile) RFCs.setup("relate", LayeredConfig(Defaults(d.get_default_options()))) for basefile in d.store.list_basefiles_for("relate"): d.relate(basefile) RFCs.teardown("relate", LayeredConfig(Defaults(d.get_default_options()))) manager.makeresources([d]) for basefile in d.store.list_basefiles_for("generate"): d.generate(basefile) d.toc() d.news() manager.frontpage([d]) shutil.rmtree("data") return_value = True
# begin example from layeredconfig import LayeredConfig, PyFile, Defaults from datetime import date, datetime conf = LayeredConfig( Defaults({ 'home': '/tmp/myapp', 'name': 'MyApp', 'dostuff': False, 'times': 4, 'duedate': date(2014, 10, 30), 'things': ['Huey', 'Dewey', 'Louie'], 'submodule': { 'retry': False, 'lastrun': datetime(2014, 10, 30, 16, 40, 22) } }), PyFile("conf.py")) # end example from datetime import date, datetime import os assert conf.home == os.getcwd() assert conf.name == 'My App' assert conf.dostuff is True assert conf.times == 4 assert conf.duedate == date.today() assert conf.things == ['Huey', 'Dewey', 'Louie'] assert conf.submodule.lastrun == datetime(2014, 10, 30, 16, 40, 22) assert conf.submodule.retry is True return_value = conf.name
def test_write_noconfigfile(self): cfg = LayeredConfig(Defaults({'lastrun': datetime(2012, 9, 18, 15, 41, 0)})) cfg.lastrun = datetime(2013, 9, 18, 15, 41, 0) LayeredConfig.write(cfg)
from layeredconfig import LayeredConfig, PyFile # begin example conf = LayeredConfig(PyFile("defaults.py"), PyFile("conf.py")) # end example from datetime import date, datetime import os assert conf.home == os.getcwd() assert conf.name == 'My App' assert conf.dostuff is True assert conf.times == 4 assert conf.duedate == date.today() assert conf.things == ['Huey', 'Dewey', 'Louie'] assert conf.submodule.lastrun == datetime(2014, 10, 30, 16, 40, 22) assert conf.submodule.retry is True return_value = conf.name
def test_typed_inifile(self): cfg = LayeredConfig(Defaults(self.types), INIFile("complex.ini")) self.supported_types = (str, bool, int, list, date, datetime) self.supports_nesting = False self._test_config_subsections(cfg)
delay = date.today() - cfg.duedate # date if cfg.dostuff: # bool for i in range(cfg.times): # int print(", ".join(cfg.things)) # list # end usetyping # begin usesubconfig subcfg = cfg.submodule if subcfg.retry: print(subcfg.lastrun.isoformat()) # end usesubconfig try: print(subcfg.home) except AttributeError: pass # begin usecascade cfg = LayeredConfig(mydefaults, myinifile, myenv, mycmdline, cascade=True) subcfg = cfg.submodule print(subcfg.home) # prints '/opt/myapp', from Commandline source root section # end usecascade assert subcfg.home == '/opt/myapp' # begin writeconfig subcfg.lastrun = datetime.now() LayeredConfig.write(cfg) # end writeconfig return_value = True
inifile = INIFile("myapp.ini") # end inifile # begin argparse parser = argparse.ArgumentParser("This is a simple program") parser.add_argument("--home", help="The home directory of the app") parser.add_argument('--dostuff', action="store_true", help="Do some work") parser.add_argument("-t", "--times", type=int, help="Number of times to do it") parser.add_argument('--things', action="append", help="Extra things to crunch") parser.add_argument('--retry', action="store_true", help="Try again") parser.add_argument("file", metavar="FILE", help="The filename to process") # end argparse # begin layeredconfig sys.argv = ['./myapp.py', '--home=/opt/myapp', '-t=2', '--dostuff', 'file.txt'] cfg = LayeredConfig(defaults, inifile, Commandline(parser=parser)) print("Starting %s in %s for %r times (doing work: %s)" % (cfg.name, cfg.home, cfg.times, cfg.dostuff)) # should print "Starting MyApp in /opt/myapp for 2 times (doing work: True)" # end layeredconfig # begin showhelp sys.argv = ['./myapp.py', '-h'] cfg = LayeredConfig(defaults, inifile, Commandline(parser=parser)) # end showhelp # begin nodefaults # NOTE: we never reach this because the previous call to -h will have # called sys.exit
def test_persistant_subrepo_config(self): self.repo.custom("set", "blahonga") LayeredConfig.write(self.repo.config) self.repo = self._createrepo() self.assertEqual("SubrepoBSubclass: blahonga", self.repo.custom("get"))
def test_modified(self): defaults = {'lastdownload': None} cfg = LayeredConfig(Defaults(defaults)) now = datetime.now() cfg.lastdownload = now self.assertEqual(cfg.lastdownload, now)
def test_newint(self): os.environ['FERENDA_DOWNLOADMAX'] = '3' config = LayeredConfig(Defaults({'downloadmax': int}), Environment(prefix="FERENDA_")) self.assertEqual(3, config.downloadmax) self.assertIsInstance(config.downloadmax, int)