def test_write(self):
        cfg = LayeredConfig(INIFile("complex.ini"))
        cfg.mymodule.expires = date(2014, 10, 24)
        cfg.mymodule.extra = ['foo', 'baz', 'quux']
        # calling write for any submodule will force a write of the
        # entire config file
        LayeredConfig.write(cfg.mymodule)
        want = """[__root__]
home = mydata
processes = 4
force = True
extra = foo, bar

[mymodule]
force = False
extra = foo, baz, quux
expires = 2014-10-24

[extramodule]
unique = True

"""
        with open("complex.ini") as fp:
            got = fp.read().replace("\r\n", "\n")
        self.assertEqual(want, got)
    def test_inifile_nonexistent(self):
        logging.getLogger().setLevel(logging.CRITICAL)
        cfg = LayeredConfig(INIFile("nonexistent.ini"))
        self.assertEqual([], list(cfg))

        # make sure a nonexistent inifile doesn't interfere with the
        # rest of the LayeredConfig object
        defobj = Defaults({'datadir': 'something'})
        iniobj = INIFile("nonexistent.ini")
        cfg = LayeredConfig(defobj, iniobj)
        self.assertEqual("something", cfg.datadir)

        # and make sure it's settable (should set up the INIFile
        # object and affect it, and leave the defaults dict untouched
        # as it's the lowest priority)
        cfg.datadir = "else"
        self.assertEqual("else", cfg.datadir)
        self.assertEqual("else", iniobj.get("datadir"))
        self.assertEqual("something", defobj.get("datadir"))

        # same as above, but with a "empty" INIFile object
        iniobj = INIFile()
        cfg = LayeredConfig(defobj, iniobj)
        self.assertEqual("something", cfg.datadir)
        cfg.datadir = "else"
        self.assertEqual("else", cfg.datadir)
    def test_write(self):
        self.maxDiff = None
        cfg = LayeredConfig(self.complex)
        cfg.mymodule.expires = date(2014, 10, 24)
        # calling write for any submodule will force a write of the
        # entire config file
        LayeredConfig.write(cfg.mymodule)
        want = """{
    "extra": [
        "foo",
        "bar"
    ],
    "extramodule": {
        "unique": true
    },
    "force": true,
    "home": "mydata",
    "mymodule": {
        "arbitrary": {
            "nesting": {
                "depth": "works"
            }
        },
        "expires": "2014-10-24",
        "extra": [
            "foo",
            "baz"
        ],
        "force": false
    },
    "processes": 4
}"""
        with open("complex.json") as fp:
            got = fp.read().replace("\r\n", "\n")
        self.assertEqual(want, got)
    def test_write(self):
        cfg = LayeredConfig(self.complex)
        cfg.mymodule.expires = date(2014, 10, 24)
        # calling write for any submodule will force a write of the
        # entire config file
        LayeredConfig.write(cfg.mymodule)
        # note that pyyaml sorts keys alphabetically and has specific
        # ideas on how to format the result (controllable through
        # mostly-undocumented args to dump())
        want = """
extra:
- foo
- bar
extramodule:
  unique: true
force: true
home: mydata
mymodule:
  arbitrary:
    nesting:
      depth: works
  expires: 2014-10-24
  extra:
  - foo
  - baz
  force: false
processes: 4
""".lstrip()
        with open("complex.yaml") as fp:
            got = fp.read().replace("\r\n", "\n")
        self.assertEqual(want, got)
Beispiel #5
0
 def download(self, basefile=None):
     if self.config.ipbasedurls:
         self._make_ipbasedurls()
     urlmap_path = self.store.path("urls", "downloaded", ".map",
                                   storage_policy="file")
     self.urlmap = {}
     if os.path.exists(urlmap_path):
         with codecs.open(urlmap_path, encoding="utf-8") as fp:
             for line in fp:
                 url, attachment = line.split("\t")
                 self.urlmap[url] = attachment.strip()
     if basefile:
         return super(PropTrips, self).download(basefile)
     try:
         now = datetime.now()
         if ('lastyear' in self.config and
                 self.config.lastyear and
                 not self.config.refresh):
             maxyear = "%s/%s" % (now.year, (now.year + 1) % 100)
             while self.config.lastyear != maxyear:
                 r = self.inner_download() 
         else:
             self.config.lastyear = ''
             r = self.inner_download()
         self.config.lastyear = "%s/%s" % (now.year - 1,
                                           (now.year % 100))
         LayeredConfig.write(self.config)     # assume we have data to write
         return r
     finally:
         with codecs.open(urlmap_path, "w", encoding="utf-8") as fp:
             for url, attachment in self.urlmap.items():
                 fp.write("%s\t%s\n" % (url, attachment))
Beispiel #6
0
 def _make_files(self, option, filedir, combinefile=None, combinefunc=None):
     urls = []
     buf = BytesIO()
     processed = set()
     # eg. self.config.cssfiles
     if getattr(self.config, option):  # it's possible to set eg
                                       # cssfiles=None when
                                       # creating the Resources
                                       # object
         for f in getattr(self.config, option):
             urls.append(self._process_file(f, buf, filedir, "ferenda.ini"))
             processed.add(f)
     for repo in self.repos:
         # FIXME: create a more generic way of optionally
         # signalling to a repo that "Hey, now it's time to create
         # your resources if you can"
         if repo.__class__.__name__ == "SFS" and option == "imgfiles":
             self.log.info("calling into SFS._makeimages()")
             LayeredConfig.set(repo.config, 'imgfiles', repo._makeimages())
         for f in getattr(repo.config, option):
             if f in processed:
                 continue
             urls.append(self._process_file(f, buf, filedir, repo.alias))
             processed.add(f)
     urls = list(filter(None, urls))
     if combinefile:
         txt = buf.getvalue().decode('utf-8')
         util.writefile(combinefile, combinefunc(txt))
         return [self._filepath_to_urlpath(combinefile, 2)]
     else:
         return urls
    def test_write(self):
        self.maxDiff = None
        cfg = LayeredConfig(self.complex)
        cfg.mymodule.expires = date(2014, 10, 24)
        cfg.mymodule.extra.append('quux')
        # calling write for any submodule will force a write of the
        # entire config file
        LayeredConfig.write(cfg.mymodule)
        # note: plistlib creates files with tabs, not spaces.
        want = """<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
	<key>extra</key>
	<array>
		<string>foo</string>
		<string>bar</string>
	</array>
	<key>extramodule</key>
	<dict>
		<key>unique</key>
		<true/>
	</dict>
	<key>force</key>
	<true/>
	<key>home</key>
	<string>mydata</string>
	<key>mymodule</key>
	<dict>
		<key>arbitrary</key>
		<dict>
			<key>nesting</key>
			<dict>
				<key>depth</key>
				<string>works</string>
			</dict>
		</dict>
		<key>expires</key>
		<string>2014-10-24</string>
		<key>extra</key>
		<array>
			<string>foo</string>
			<string>baz</string>
			<string>quux</string>
		</array>
		<key>force</key>
		<false/>
	</dict>
	<key>processes</key>
	<integer>4</integer>
</dict>
</plist>
"""
        if sys.version_info < (2,7,0): # pragma: no cover
            # on py26, the doctype includes "Apple Computer" not "Apple"...
            want = want.replace("//Apple//", "//Apple Computer//")
        with open("complex.plist") as fp:
            got = fp.read().replace("\r\n", "\n")
        self.assertEqual(want, got)
 def test_get(self):
     cfg = LayeredConfig(Defaults({'codedefaults': 'yes',
                                   'force': False,
                                   'home': '/usr/home'}),
                         INIFile('simple.ini'))
     # and then do a bunch of get() calls with optional fallbacks
     self.assertEqual("yes", LayeredConfig.get(cfg, "codedefaults"))
     self.assertEqual("mydata", LayeredConfig.get(cfg, "home"))
     self.assertEqual(None, LayeredConfig.get(cfg, "nonexistent"))
     self.assertEqual("NO!", LayeredConfig.get(cfg, "nonexistent", "NO!"))
    def test_set_novalue(self):
        # it should be possible to set values that are defined in any
        # of the configsources, even though only typing information
        # exists there.
        cfg = LayeredConfig(Defaults({'placeholder': int}),
                            Commandline([]))
        cfg.placeholder = 42

        # but it shouldn't be possible to set values that hasn't been
        # defined anywhere.
        with self.assertRaises(AttributeError):
            cfg.nonexistent = 43
 def test_set(self):
     # a value is set in a particular underlying source, and the
     # dirty flag isn't set.
     cfg = LayeredConfig(INIFile("simple.ini"))
     LayeredConfig.set(cfg, 'expires', date(2013, 9, 18),
                       "inifile")
     # NOTE: For this config, where no type information is
     # available for key 'expires', INIFile.set will convert the
     # date object to a string, at which point typing is lost.
     # Therefore this commmented-out test will fail
     # self.assertEqual(date(2013, 9, 18), cfg.expires)
     self.assertEqual("2013-09-18", cfg.expires)
     self.assertFalse(cfg._sources[0].dirty)
Beispiel #11
0
 def test_typed_override(self):
     # make sure this auto-typing isn't run for bools
     types = {'logfile': True}
     cmdline = ["--logfile=out.log"]
     cfg = LayeredConfig(Defaults(types), Commandline(cmdline))
     self.assertEqual(cfg.logfile, "out.log")
    def test_write(self):
        def indexfilter(node):
            # remove keys like createdIdex / modifiedIndex whose
            # values always changes
            if isinstance(node, dict):
                for key in list(node.keys()):
                    if key == "nodes":
                        indexfilter(node[key])
                    elif key.endswith("Index"):
                        del node[key]
            else:
                node[:] = sorted(node, key=itemgetter('key'))
                for subnode in node:
                    indexfilter(subnode)

        cfg = LayeredConfig(self.complex)
        cfg.mymodule.expires = date(2014, 10, 24)
        LayeredConfig.write(cfg.mymodule)
        want = """
{
    "dir": true,
    "key": "/",
    "nodes": [
        {
            "createdIndex": 4627,
            "key": "/home",
            "modifiedIndex": 4627,
            "value": "mydata"
        },
        {
            "createdIndex": 4628,
            "key": "/processes",
            "modifiedIndex": 4628,
            "value": "4"
        },
        {
            "createdIndex": 4629,
            "key": "/force",
            "modifiedIndex": 4629,
            "value": "true"
        },
        {
            "createdIndex": 4630,
            "key": "/extra",
            "modifiedIndex": 4630,
            "value": "foo, bar"
        },
        {
            "createdIndex": 4631,
            "dir": true,
            "key": "/mymodule",
            "modifiedIndex": 4631,
            "nodes": [
                {
                    "createdIndex": 4631,
                    "key": "/mymodule/force",
                    "modifiedIndex": 4631,
                    "value": "false"
                },
                {
                    "createdIndex": 4632,
                    "key": "/mymodule/extra",
                    "modifiedIndex": 4632,
                    "value": "foo, baz"
                },
                {
                    "createdIndex": 4633,
                    "key": "/mymodule/expires",
                    "modifiedIndex": 4633,
                    "value": "2014-10-24"
                },
                {
                    "createdIndex": 4634,
                    "dir": true,
                    "key": "/mymodule/arbitrary",
                    "modifiedIndex": 4634,
                    "nodes": [
                        {
                            "createdIndex": 4634,
                            "dir": true,
                            "key": "/mymodule/arbitrary/nesting",
                            "modifiedIndex": 4634,
                            "nodes": [
                                {
                                    "createdIndex": 4634,
                                    "key": "/mymodule/arbitrary/nesting/depth",
                                    "modifiedIndex": 4634,
                                    "value": "works"
                                }
                            ]
                        }
                    ]
                }
            ]
        },
        {
            "createdIndex": 4635,
            "dir": true,
            "key": "/extramodule",
            "modifiedIndex": 4635,
            "nodes": [
                {
                    "createdIndex": 4635,
                    "key": "/extramodule/unique",
                    "modifiedIndex": 4635,
                    "value": "true"
                }
            ]
        }
    ]
}"""
        want = json.loads(want)
        indexfilter(want)
        got = requests.get("http://localhost:4001/v2/keys/?recursive=true").json()['node']
        indexfilter(got)
        self.assertEqual(want, got)
Beispiel #13
0
myinifile = INIFile("myapp.ini")
# end inifile

# begin environment
env = {'MYAPP_HOME': 'C:\\Progra~1\\MyApp', 'MYAPP_SUBMODULE_RETRY': 'True'}
myenv = Environment(env, prefix="MYAPP_")
# end environment

# begin commandline
mycmdline = Commandline(['-f', '--home=/opt/myapp', '--times=2', '--dostuff'])
rest = mycmdline.rest
# end commandline
assert rest == ['-f']

# begin makeconfig
cfg = LayeredConfig(mydefaults, myinifile, myenv, mycmdline)
# end makeconfig
import os


def do_stuff(action, idx):
    pass


# begin useconfig
print("%s starting, home in %s" % (cfg.name, cfg.home))
# end useconfig

# begin usetyping
delay = date.today() - cfg.duedate  # date
if cfg.dostuff:  # bool
Beispiel #14
0
            items,
            'doccount':
            len(list(self.store.list_basefiles_for("_postgenerate")))
        })


# end frontpage_content

from ferenda import manager
from layeredconfig import LayeredConfig, Defaults
import sys
manager.setup_logger("DEBUG")
d = RFCs(downloadmax=5)

d.download()
for basefile in d.store.list_basefiles_for("parse"):
    d.parse(basefile)
RFCs.setup("relate", LayeredConfig(Defaults(d.get_default_options())))
for basefile in d.store.list_basefiles_for("relate"):
    d.relate(basefile)
RFCs.teardown("relate", LayeredConfig(Defaults(d.get_default_options())))
manager.makeresources([d])
for basefile in d.store.list_basefiles_for("generate"):
    d.generate(basefile)
d.toc()

d.news()
manager.frontpage([d])
shutil.rmtree("data")
return_value = True
# begin example
from layeredconfig import LayeredConfig, PyFile, Defaults
from datetime import date, datetime

conf = LayeredConfig(
    Defaults({
        'home': '/tmp/myapp',
        'name': 'MyApp',
        'dostuff': False,
        'times': 4,
        'duedate': date(2014, 10, 30),
        'things': ['Huey', 'Dewey', 'Louie'],
        'submodule': {
            'retry': False,
            'lastrun': datetime(2014, 10, 30, 16, 40, 22)
        }
    }), PyFile("conf.py"))
# end example
from datetime import date, datetime
import os
assert conf.home == os.getcwd()
assert conf.name == 'My App'
assert conf.dostuff is True
assert conf.times == 4
assert conf.duedate == date.today()
assert conf.things == ['Huey', 'Dewey', 'Louie']
assert conf.submodule.lastrun == datetime(2014, 10, 30, 16, 40, 22)
assert conf.submodule.retry is True

return_value = conf.name
 def test_write_noconfigfile(self):
     cfg = LayeredConfig(Defaults({'lastrun':
                                   datetime(2012, 9, 18, 15, 41, 0)}))
     cfg.lastrun = datetime(2013, 9, 18, 15, 41, 0)
     LayeredConfig.write(cfg)
from layeredconfig import LayeredConfig, PyFile

# begin example
conf = LayeredConfig(PyFile("defaults.py"), PyFile("conf.py"))
# end example
from datetime import date, datetime
import os
assert conf.home == os.getcwd()
assert conf.name == 'My App'
assert conf.dostuff is True
assert conf.times == 4
assert conf.duedate == date.today()
assert conf.things == ['Huey', 'Dewey', 'Louie']
assert conf.submodule.lastrun == datetime(2014, 10, 30, 16, 40, 22)
assert conf.submodule.retry is True

return_value = conf.name
Beispiel #18
0
 def test_typed_inifile(self):
     cfg = LayeredConfig(Defaults(self.types), INIFile("complex.ini"))
     self.supported_types = (str, bool, int, list, date, datetime)
     self.supports_nesting = False
     self._test_config_subsections(cfg)
Beispiel #19
0
delay = date.today() - cfg.duedate  # date
if cfg.dostuff: # bool
    for i in range(cfg.times):  # int
        print(", ".join(cfg.things))  # list
# end usetyping

# begin usesubconfig
subcfg = cfg.submodule  
if subcfg.retry:
    print(subcfg.lastrun.isoformat())
# end usesubconfig

try:
    print(subcfg.home)
except AttributeError:
    pass

# begin usecascade
cfg = LayeredConfig(mydefaults, myinifile, myenv, mycmdline, cascade=True)
subcfg = cfg.submodule
print(subcfg.home)  # prints '/opt/myapp', from Commandline source root section
# end usecascade
assert subcfg.home == '/opt/myapp'

# begin writeconfig
subcfg.lastrun = datetime.now()
LayeredConfig.write(cfg)
# end writeconfig

return_value = True
inifile = INIFile("myapp.ini")
# end inifile

# begin argparse
parser = argparse.ArgumentParser("This is a simple program")
parser.add_argument("--home", help="The home directory of the app")
parser.add_argument('--dostuff', action="store_true", help="Do some work")
parser.add_argument("-t", "--times", type=int, help="Number of times to do it")
parser.add_argument('--things', action="append", help="Extra things to crunch")
parser.add_argument('--retry', action="store_true", help="Try again")
parser.add_argument("file", metavar="FILE", help="The filename to process")
# end argparse

# begin layeredconfig
sys.argv = ['./myapp.py', '--home=/opt/myapp', '-t=2', '--dostuff', 'file.txt']
cfg = LayeredConfig(defaults, inifile, Commandline(parser=parser))
print("Starting %s in %s for %r times (doing work: %s)" %
      (cfg.name, cfg.home, cfg.times, cfg.dostuff))
# should print "Starting MyApp in /opt/myapp for 2 times (doing work: True)"
# end layeredconfig

# begin showhelp
sys.argv = ['./myapp.py', '-h']
cfg = LayeredConfig(defaults, inifile, Commandline(parser=parser))
# end showhelp

# begin nodefaults

# NOTE: we never reach this because the previous call to -h will have
# called sys.exit
Beispiel #21
0
 def test_persistant_subrepo_config(self):
     self.repo.custom("set", "blahonga")
     LayeredConfig.write(self.repo.config)
     self.repo = self._createrepo()
     self.assertEqual("SubrepoBSubclass: blahonga", self.repo.custom("get"))
 def test_modified(self):
     defaults = {'lastdownload': None}
     cfg = LayeredConfig(Defaults(defaults))
     now = datetime.now()
     cfg.lastdownload = now
     self.assertEqual(cfg.lastdownload, now)
Beispiel #23
0
 def test_newint(self):
     os.environ['FERENDA_DOWNLOADMAX'] = '3'
     config = LayeredConfig(Defaults({'downloadmax': int}),
                            Environment(prefix="FERENDA_"))
     self.assertEqual(3, config.downloadmax)
     self.assertIsInstance(config.downloadmax, int)