def __init__(self, input_path, **kw):
# setting up paths
self.paths = OMD()
self._paths = OMD() # for the raw input paths
self.fal = ChertFAL(chlog)
set_path = self._set_path
set_path('input_path', input_path)
set_path('config_path', kw.pop('config_path', None),
DEFAULT_CONFIG_FILENAME)
set_path('entries_path', kw.pop('entries_path', None), 'entries')
set_path('themes_path', kw.pop('themes_path', None), 'themes')
set_path('uploads_path',
kw.pop('uploads_path', None),
'uploads',
required=False)
set_path('output_path',
kw.pop('output_path', None),
'site',
required=False)
self.reload_config()
self.reset()
self.dev_mode = kw.pop('dev_mode', False)
if kw:
raise TypeError('unexpected keyword arguments: %r' % kw)
chlog.debug('init site').success()
return
def test_merge_omd():
target = [{'a': 'A'}, {'a': 'aleph'}]
assert glom(target, Merge(init=OMD)) == OMD({'a': 'aleph'})
assert glom(target, Merge(init=OMD,
op='update_extend')) == OMD([('a', 'A'),
('a', 'aleph')])
def main():
cities_json = json.load(open(CITIES_FILE))
pop_cities = sorted(cities_json,
lambda o, _: int(o['population']),
reverse=True)
# print pop_cities[:20]
cities = [o['city'] for o in pop_cities][:20]
jsonl_iter = JSONLIterator(open(BRIEFINGS_FILE))
res = defaultdict(list)
for obj in jsonl_iter:
title = obj['title']
briefing_html = obj['content']
content_tree = soupparser.fromstring(briefing_html)
pane_tree = content_tree.cssselect('.pane-node-field-forall-body')
briefing_text = pane_tree[0].text_content()
for city in cities:
if city in briefing_text:
res[city].append(title)
print 'found', repr(city), 'in', repr(title)
omd = OMD()
for k in res:
omd.addlist(k, res[k])
top_items = sorted(omd.counts().items(), key=lambda x: x[1], reverse=True)
import pdb
pdb.set_trace()
def test_setdefault():
omd = OMD()
empty_list = []
x = omd.setdefault('1', empty_list)
assert x is empty_list
y = omd.setdefault('2')
assert y is None
assert omd.setdefault('1', None) is empty_list
def test_setdefault():
omd = OMD()
empty_list = []
x = omd.setdefault("1", empty_list)
assert x is empty_list
y = omd.setdefault("2")
assert y is None
assert omd.setdefault("1", None) is empty_list
def test_setdefault():
omd = OMD()
empty_list = []
x = omd.setdefault('1', empty_list)
assert x is empty_list
y = omd.setdefault('2')
assert y is None
assert omd.setdefault('1', None) is empty_list
def get_all_secret_names(self):
"return a map of secret names to names of domains that contain that secret"
res = OMD()
for domain_name, domain in self._domains.items():
secrets_dict = domain._secrets
for secret_name in secrets_dict:
res.add(secret_name, domain_name)
return dict(res)
def test_copy():
for itemset in _ITEMSETS:
omd = OMD(itemset)
omd_c = omd.copy()
assert omd == omd_c
if omd_c:
omd_c.pop(itemset[0][0])
assert omd != omd_c
return
def test_invert():
for items in _ITEMSETS:
omd = OMD(items)
iomd = omd.inverted()
# first, test all items made the jump
assert len(omd.items(multi=True)) == len(iomd.items(multi=True))
for val in omd.values():
assert val in iomd # all values present as keys
def test_copy():
for itemset in _ITEMSETS:
omd = OMD(itemset)
omd_c = omd.copy()
assert omd == omd_c
if omd_c:
omd_c.pop(itemset[0][0])
assert omd != omd_c
return
def test_invert():
for items in _ITEMSETS:
omd = OMD(items)
iomd = omd.inverted()
# first, test all items made the jump
assert len(omd.items(multi=True)) == len(iomd.items(multi=True))
for val in omd.values():
assert val in iomd # all values present as keys
def update_projects(self, projects):
to_update = OMD([(slugify(p.name), p) for p in projects])
new_list = []
for proj in self.project_list:
if proj.name_slug not in to_update:
new_list.append(proj)
continue
new_list.append(to_update.pop(proj.name_slug))
new_list.extend(to_update.values())
self.project_list = new_list
def test_reversed():
try:
from collections import OrderedDict
except:
# skip on python 2.6
return
for items in _ITEMSETS:
omd = OMD(items)
od = OrderedDict(items)
for ik, ok in zip(reversed(od), reversed(omd)):
assert ik == ok
r100 = range(100)
omd = OMD(zip(r100, r100))
for i in r100:
omd.add(i, i)
r100 = list(reversed(r100))
assert list(reversed(omd)) == r100
omd = OMD()
assert list(reversed(omd)) == list(reversed(omd.keys()))
for i in range(20):
for j in range(i):
omd.add(i, i)
assert list(reversed(omd)) == list(reversed(omd.keys()))
def test_dict_init():
d = dict(_ITEMSETS[1])
omd = OMD(d)
assert omd["a"] == 1
assert omd["b"] == 2
assert omd["c"] == 3
assert len(omd) == 3
assert omd.getlist("a") == [1]
assert omd == d
def __init__(self, project_list, tagsonomy):
self.project_list = []
self.tagsonomy = tagsonomy
self.tag_registry = OMD()
self.tag_alias_map = OMD()
for tag in self.tagsonomy['topic']:
self.register_tag('topic', tag)
for project in project_list:
self.project_list.append(Project.from_dict(project))
def test_dict_init():
d = dict(_ITEMSETS[1])
omd = OMD(d)
assert omd['a'] == 1
assert omd['b'] == 2
assert omd['c'] == 3
assert len(omd) == 3
assert omd.getlist('a') == [1]
assert omd == d
def test_dict_init():
d = dict(_ITEMSETS[1])
omd = OMD(d)
assert omd['a'] == 1
assert omd['b'] == 2
assert omd['c'] == 3
assert len(omd) == 3
assert omd.getlist('a') == [1]
assert omd == d
def test_kv_consistency():
for itemset in _ITEMSETS:
omd = OMD(itemset)
for multi in (True, False):
items = omd.items(multi=multi)
keys = omd.keys(multi=multi)
values = omd.values(multi=multi)
assert keys == [x[0] for x in items]
assert values == [x[1] for x in items]
return
def test_kv_consistency():
for itemset in _ITEMSETS:
omd = OMD(itemset)
for multi in (True, False):
items = omd.items(multi=multi)
keys = omd.keys(multi=multi)
values = omd.values(multi=multi)
assert keys == [x[0] for x in items]
assert values == [x[1] for x in items]
return
def test_setdefault():
omd = OMD()
empty_list = []
x = omd.setdefault('1', empty_list)
assert x is empty_list
y = omd.setdefault('2')
assert y is None
assert omd.setdefault('1', None) is empty_list
e_omd = OMD()
e_omd.addlist(1, [])
assert e_omd.popall(1, None) is None
assert len(e_omd) == 0
def __init__(self, project_list, tagsonomy):
self.project_list = []
self.tagsonomy = tagsonomy
self.tag_registry = OMD()
for tag_group in ('topic', 'platform'): # TODO: framework, license
for tag in self.tagsonomy[tag_group]:
self.register_tag(tag_group, tag)
for project in project_list:
new_tags = soft_sorted(project.get('tags', []), first=self.tag_registry.keys())
project['tags'] = new_tags
self.project_list.append(Project.from_dict(project))
def test_multi_correctness():
size = 100
redun = 5
_rng = range(size)
_rng_redun = list(range(size//redun)) * redun
_pairs = zip(_rng_redun, _rng)
omd = OMD(_pairs)
for multi in (True, False):
vals = [x[1] for x in omd.iteritems(multi=multi)]
strictly_ascending = all([x < y for x, y in zip(vals, vals[1:])])
assert strictly_ascending
return
def __init__(self, **kwargs):
access = kwargs.get('accesskey', None)
secret = kwargs.get('secret', None)
bucket = kwargs.get('bucket', '')
configpath = kwargs.get('configpath', None)
omd = OMD()
omd.update({'accesskey': access, 'secret': secret, 'bucket': bucket})
if configpath is not None and exists(configpath):
with open(configpath, 'r') as f:
omd.update_extend(yaml.safe_load(f))
self.access = omd.get('accesskey', access)
self.secret = omd.get('secret', secret)
bucket_name = omd.get('bucket', bucket)
cd = None
try:
self.s3 = boto3.client('s3',
aws_access_key_id=self.access,
aws_secret_access_key=self.secret)
self.bucket = boto3.resource(
's3',
aws_access_key_id=self.access,
aws_secret_access_key=self.secret).Bucket(bucket_name)
cd = self.bucket.creation_date
# make sure you actually have a bucket
except ClientError as e:
raise ValueError(e.response['Error']['Message'])
if cd is None:
raise ValueError("This bucket [" + self.bucket.name +
"] does not exist")
logging.getLogger("connectionpool.py").setLevel(logging.WARNING)
logging.getLogger("connectionpool").setLevel(logging.WARNING)
def test_eq():
omd = OMD(_ITEMSETS[3])
assert omd == omd
assert not (omd != omd)
omd2 = OMD(_ITEMSETS[3])
assert omd == omd2
assert omd2 == omd
assert not (omd != omd2)
d = dict(_ITEMSETS[3])
assert d == omd
omd3 = OMD(d)
assert omd != omd3
def test_multi_correctness():
size = 100
redun = 5
_rng = range(size)
_rng_redun = list(range(size // redun)) * redun
_pairs = zip(_rng_redun, _rng)
omd = OMD(_pairs)
for multi in (True, False):
vals = [x[1] for x in omd.iteritems(multi=multi)]
strictly_ascending = all([x < y for x, y in zip(vals, vals[1:])])
assert strictly_ascending
return
def _parse_flags(self, cmd_flag_map, args):
"""Expects arguments after the initial command and subcommands (i.e.,
the second item returned from _parse_subcmds)
Returns a tuple of (multidict of flag names to parsed and validated values, remaining_args).
Raises on unknown subcommands.
"""
flag_value_map = OMD()
ff_path_res_map = OrderedDict()
ff_path_seen = set()
orig_args = args
while args:
arg = args[0]
if not arg or arg[0] != '-' or arg == '-' or arg == '--':
# posargs or post_posargs beginning ('-' is a conventional pos arg for stdin)
break
flag, value, args = self._parse_single_flag(cmd_flag_map, args)
flag_value_map.add(flag.name, value)
if flag is self.flagfile_flag:
self._parse_flagfile(cmd_flag_map,
value,
res_map=ff_path_res_map)
for path, ff_flag_value_map in ff_path_res_map.items():
if path in ff_path_seen:
continue
flag_value_map.update_extend(ff_flag_value_map)
ff_path_seen.add(path)
return flag_value_map, ff_path_res_map, args
class ProjectList(object):
def __init__(self, project_list, tagsonomy):
self.project_list = []
self.tagsonomy = tagsonomy
self.tag_registry = OMD()
self.tag_alias_map = OMD()
for tag in self.tagsonomy['topic']:
self.register_tag('topic', tag)
for project in project_list:
self.project_list.append(Project.from_dict(project))
@classmethod
def from_path(cls, path):
data = yaml.safe_load(open(path))
return cls(data['projects'], data['tagsonomy'])
def register_tag(self, tag_type, tag_entry, tag_path=()):
if isinstance(tag_entry, str):
tag, tag_entry = tag_entry, {}
else:
tag, tag_entry = _unwrap_dict(tag_entry)
tag_entry = dict(tag_entry)
tag_entry['tag'] = tag
tag_entry['tag_type'] = tag_type
if not tag_entry.get('title'):
tag_entry["title"] = tag.replace('_', ' ').title()
subtags = []
for subtag_entry in tag_entry.pop('subtags', []):
st = self.register_tag(
tag_type,
subtag_entry,
tag_path=(tag, ) if not tag_path else tag_path + (tag, ))
subtags.append(st)
tag_entry['subtags'] = tuple(subtags)
tag_entry['fq_tag'] = '.'.join(tag_path + (tag, ))
if not tag_path:
ret = TagEntry(**tag_entry)
else:
ret = TagEntry(tag_path=tag_path, **tag_entry)
self.tag_registry[tag] = ret
return ret
def get_projects_by_type(self, type_name):
ret = OMD()
for tag, tag_entry in self.tag_registry.items():
if tag_entry.tag_type != type_name:
continue
ret[tag_entry] = []
for project in self.project_list:
if tag in project.tags:
ret[tag_entry].append(project)
ret[tag_entry].sort(key=lambda x: x.name)
return ret
def test_update_basic():
omd = OMD(_ITEMSETS[1])
omd2 = OMD({'a': 10})
omd.update(omd2)
assert omd['a'] == 10
assert omd.getlist('a') == [10]
omd2_c = omd2.copy()
omd2_c.pop('a')
assert omd2 != omd2_c
def test_update_extend():
for first, second in zip(_ITEMSETS, _ITEMSETS[1:] + [[]]):
omd1 = OMD(first)
omd2 = OMD(second)
ref = dict(first)
orig_keys = set(omd1)
ref.update(second)
omd1.update_extend(omd2)
for k in omd2:
assert len(omd1.getlist(k)) >= len(omd2.getlist(k))
assert omd1.todict() == ref
assert orig_keys <= set(omd1)
def test_types():
try:
from collections.abc import MutableMapping
except ImportError:
from collections import MutableMapping
omd = OMD()
assert isinstance(omd, dict)
assert isinstance(omd, MutableMapping)
def test_todict():
omd = OMD(_ITEMSETS[2])
assert len(omd) == 1
assert omd['A'] == 'One'
d = dict(omd)
assert len(d) == 1
assert d['A'] == ['One', 'One', 'One']
flat = omd.todict()
assert flat['A'] == 'One'
for itemset in _ITEMSETS:
omd = OMD(itemset)
d = dict(itemset)
flat = omd.todict()
assert flat == d
def test_todict():
omd = OMD(_ITEMSETS[2])
assert len(omd) == 1
assert omd["A"] == "One"
d = dict(omd)
assert len(d) == 1
assert d["A"] == ["One", "One", "One"]
flat = omd.todict()
assert flat["A"] == "One"
for itemset in _ITEMSETS:
omd = OMD(itemset)
d = dict(itemset)
flat = omd.todict()
assert flat == d
def get_projects_by_type(self, type_name):
ret = OMD()
for tag, tag_entry in self.tag_registry.items():
if tag_entry.tag_type != type_name:
continue
ret[tag_entry] = []
for project in self.project_list:
if tag in project.tags:
ret[tag_entry].append(project)
return ret
def test_clear():
for itemset in _ITEMSETS:
omd = OMD(itemset)
omd.clear()
assert len(omd) == 0
assert not omd
omd.clear()
assert not omd
omd['a'] = 22
assert omd
omd.clear()
assert not omd
def validate(self):
self._call_custom_hook('pre_validate')
dup_id_map = {}
eid_map = OMD([(e.entry_root, e) for e in self.entries])
for eid in eid_map:
elist = eid_map.getlist(eid)
if len(elist) > 1:
dup_id_map[eid] = elist
if dup_id_map:
raise ValueError('duplicate entry IDs detected: %r' % dup_id_map)
self._call_custom_hook('post_validate')
def test_update():
for first, second in zip(_ITEMSETS, _ITEMSETS[1:]):
omd1 = OMD(first)
omd2 = OMD(second)
ref1 = dict(first)
ref2 = dict(second)
omd1.update(omd2)
ref1.update(ref2)
assert omd1.todict() == ref1
omd1_repr = repr(omd1)
omd1.update(omd1)
assert omd1_repr == repr(omd1)
def test_clear():
for itemset in _ITEMSETS:
omd = OMD(itemset)
omd.clear()
assert len(omd) == 0
assert not omd
omd.clear()
assert not omd
omd["a"] = 22
assert omd
omd.clear()
assert not omd
def __init__(self, project_list, tagsonomy):
self.project_list = []
self.tagsonomy = tagsonomy
self.tag_registry = OMD()
for tag_group in ('topic', 'platform'): # TODO: framework, license
for tag in self.tagsonomy[tag_group]:
self.register_tag(tag_group, tag)
errors = []
for project in project_list:
new_tags = tuple(
soft_sorted(project.get('tags', []),
first=self.tag_registry.keys()))
project['tags'] = new_tags
try:
project_obj = Project.from_dict(project)
except ApatiteError as ae:
errors.append(ae)
continue
self.project_list.append(project_obj)
dupe_groups = redundant(self.project_list,
key=lambda p: slugify(p.name),
groups=True)
dupe_groups += redundant(self.project_list,
key=lambda p: p.repo_url,
groups=True)
dupe_groups = unique([tuple(dg) for dg in dupe_groups])
for group in dupe_groups:
dpe = DuplicateProjectError('ambiguous or duplicate projects: %r' %
[p.name for p in group])
errors.append(dpe)
if errors:
raise ProjectListError.from_errors(errors)
return
def get_details():
'''looks for values in pdf_store.yml '''
yaml_path = "pdf_store.yml"
omd = OMD()
# Fallback to defaults for local devserver
omd.update({'s3_yaml': 's3.yml', 'tmpdir': './tmp/', 'instance': 'dev'})
if exists(yaml_path):
with open(yaml_path, 'r') as f:
omd.update_extend(yaml.safe_load(f))
return omd
def test_update_basic():
omd = OMD(_ITEMSETS[1])
omd2 = OMD({'a': 10})
omd.update(omd2)
assert omd['a'] == 10
assert omd.getlist('a') == [10]
omd2_c = omd2.copy()
omd2_c.pop('a')
assert omd2 != omd2_c
def test_update_basic():
omd = OMD(_ITEMSETS[1])
omd2 = OMD({"a": 10})
omd.update(omd2)
assert omd["a"] == 10
assert omd.getlist("a") == [10]
omd2_c = omd2.copy()
omd2_c.pop("a")
assert omd2 != omd2_c
def test_update_extend():
for first, second in zip(_ITEMSETS, _ITEMSETS[1:] + [[]]):
omd1 = OMD(first)
omd2 = OMD(second)
ref = dict(first)
orig_keys = set(omd1)
ref.update(second)
omd1.update_extend(omd2)
for k in omd2:
assert len(omd1.getlist(k)) >= len(omd2.getlist(k))
assert omd1.todict() == ref
assert orig_keys <= set(omd1)
def _parse_flagfile(self, cmd_flag_map, path_or_file, res_map=None):
ret = res_map if res_map is not None else OrderedDict()
if callable(getattr(path_or_file, 'read', None)):
# enable StringIO and custom flagfile opening
f_name = getattr(path_or_file, 'name', None)
path = os.path.abspath(f_name) if f_name else repr(path_or_file)
ff_text = path_or_file.read()
else:
path = os.path.abspath(path_or_file)
try:
with codecs.open(path_or_file, 'r', 'utf-8') as f:
ff_text = f.read()
except (UnicodeError, EnvironmentError) as ee:
raise ArgumentParseError(
'failed to load flagfile "%s", got: %r' % (path, ee))
if path in res_map:
# we've already seen this file
return res_map
ret[path] = cur_file_res = OMD()
lines = ff_text.splitlines()
for lineno, line in enumerate(lines, 1):
try:
args = shlex.split(line, comments=True)
if not args:
continue # comment or empty line
flag, value, leftover_args = self._parse_single_flag(
cmd_flag_map, args)
if leftover_args:
raise ArgumentParseError(
'excessive flags or arguments for flag "%s",'
' expected one flag per line' % flag.name)
cur_file_res.add(flag.name, value)
if flag is self.flagfile_flag:
self._parse_flagfile(cmd_flag_map, value, res_map=ret)
except FaceException as fe:
fe.args = (fe.args[0] + ' (on line %s of flagfile "%s")' %
(lineno, path), )
raise
return ret
def test_update():
for first, second in zip(_ITEMSETS, _ITEMSETS[1:]):
omd1 = OMD(first)
omd2 = OMD(second)
ref1 = dict(first)
ref2 = dict(second)
omd1.update(omd2)
ref1.update(ref2)
assert omd1.todict() == ref1
omd1_repr = repr(omd1)
omd1.update(omd1)
assert omd1_repr == repr(omd1)
def test_todict():
omd = OMD(_ITEMSETS[2])
assert len(omd) == 1
assert omd['A'] == 'One'
d = omd.todict(multi=True)
assert len(d) == 1
assert d['A'] == ['One', 'One', 'One']
flat = omd.todict()
assert flat['A'] == 'One'
for itemset in _ITEMSETS:
omd = OMD(itemset)
d = dict(itemset)
flat = omd.todict()
assert flat == d
return
def ConfigSources(yaml_path):
'''Helper method returning an :py:class:`boltons.dictutils.OrderedMultiDict` representing configuration sources (defaults, yaml)'''
omd = OMD()
# Fallback to defaults for local devserver
omd.update({
'baseurl': 'http://localhost:4567',
'username': '******',
'password': '******'
})
if exists(yaml_path):
with open(yaml_path, 'r') as f:
omd.update_extend(yaml.safe_load(f))
log.debug("loaded yaml config", source=yaml_path)
return omd
def test_pop():
omd = OMD()
omd.add('even', 0)
omd.add('odd', 1)
omd.add('even', 2)
assert omd.pop('odd') == 1
assert omd.pop('odd', 99) == 99
try:
omd.pop('odd')
import pdb;pdb.set_trace()
assert False
except KeyError:
pass
assert len(omd) == 1
assert len(omd.items(multi=True)) == 2
def test_poplast():
for items in _ITEMSETS[1:]:
omd = OMD(items)
assert omd.poplast() == items[-1][-1]
def test_pop():
omd = OMD()
omd.add("even", 0)
omd.add("odd", 1)
omd.add("even", 2)
assert omd.pop("odd") == 1
assert omd.pop("odd", 99) == 99
try:
omd.pop("odd")
import pdb
pdb.set_trace()
assert False
except KeyError:
pass
assert len(omd) == 1
assert len(omd.items(multi=True)) == 2
def test_pop_all():
omd = OMD()
omd.add("even", 0)
omd.add("odd", 1)
omd.add("even", 2)
assert omd.popall("odd") == [1]
assert len(omd) == 1
try:
omd.popall("odd")
assert False
except KeyError:
pass
assert omd.popall("odd", None) is None
assert omd.popall("even") == [0, 2]
assert len(omd) == 0
assert omd.popall("nope", None) is None
def test_pop_all():
omd = OMD()
omd.add('even', 0)
omd.add('odd', 1)
omd.add('even', 2)
assert omd.popall('odd') == [1]
assert len(omd) == 1
try:
omd.popall('odd')
assert False
except KeyError:
pass
assert omd.popall('odd', None) is None
assert omd.popall('even') == [0, 2]
assert len(omd) == 0
assert omd.popall('nope', None) is None