def load_index_config(name, _seen=None, _first_run=[True]):
if _first_run:
_make_extra_filters()
_first_run.clear()
if _seen is None:
_seen = set()
if name in _seen:
logger.Error(
"Inherited file {} already encountered, skipping".format(name))
return out
_seen.add(name)
file = (sc.indexer_dir / name).with_suffix('.json')
out = {}
with file.open('r', encoding='utf8') as f:
try:
config = json.load(f)
except ValueError:
logger.error(
'An error occured while parsing {!s}'.format(file))
raise
for filename in config.get("inherits", []):
inherited_config = ElasticIndexer.load_index_config(
filename, _seen)
recursive_merge(out, inherited_config)
recursive_merge(out, config.get("index", {}))
return out
def load_index_config(name, _seen=None, _first_run=[True]):
if _first_run:
_make_extra_filters()
_first_run.clear()
if _seen is None:
_seen = set()
if name in _seen:
logger.Error("Inherited file {} already encountered, skipping".format(name))
return out
_seen.add(name)
file = (sc.indexer_dir / name).with_suffix('.json')
out = {}
with file.open('r', encoding='utf8') as f:
try:
config = json.load(f)
except ValueError:
logger.error('An error occured while parsing {!s}'.format(file))
raise
for filename in config.get("inherits", []):
inherited_config = ElasticIndexer.load_index_config(filename, _seen)
recursive_merge(out, inherited_config)
recursive_merge(out, config.get("index", {}))
return out
def load(path):
out = {}
for file in path.iterdir():
key = file.stem
if file.is_dir():
subtree = load(file)
elif file.suffix == '.json':
with file.open('r', encoding='UTF-8') as f:
subtree = json.load(f)
if key not in out:
out[key] = subtree
else:
recursive_merge(out[key], subtree)
return out
def load(path):
out = {}
for file in path.iterdir():
key = file.stem
if file.is_dir():
subtree = load(file)
elif file.suffix == '.json':
with file.open('r', encoding='UTF-8') as f:
subtree = json.load(f)
if key not in out:
out[key] = subtree
else:
recursive_merge(out[key], subtree)
return out
def test_recursive_merge(self):
from copy import deepcopy
a = {1: 2, 2: 4}
b = {3: 9, 4: 16}
c = {'z': deepcopy(a)}
d = {'z': deepcopy(b)}
util.recursive_merge(a, b)
self.assertEqual(a, {1: 2, 2: 4, 3: 9, 4: 16})
util.recursive_merge(c, d)
self.assertEqual(c, {'z': {1: 2, 2: 4, 3: 9, 4: 16}})
e = {
'animals': {
'pets': {
'friendly': ['dogs'],
'snobby': ['cats']
}
},
'ready': 0
}
f = {
'animals': {
'pets': {
'friendly': ['dogs', 'rats', 'hamsters'],
'wet': ['goldfish', 'eels']
},
'herbivores': {
'plains': ['elephants']
}
},
'ready': True
}
util.recursive_merge(e, f)
self.assertEqual(
e, {
'animals': {
'pets': {
'friendly': ['dogs', 'rats', 'hamsters'],
'snobby': ['cats'],
'wet': ['goldfish', 'eels']
},
'herbivores': {
'plains': ['elephants']
}
},
'ready': True
})
def test_recursive_merge(self):
from copy import deepcopy
a = {1: 2, 2: 4}
b = {3: 9, 4: 16}
c = {'z': deepcopy(a)}
d = {'z': deepcopy(b)}
util.recursive_merge(a, b)
self.assertEqual(a, {1: 2, 2:4, 3:9, 4:16})
util.recursive_merge(c, d)
self.assertEqual(c, {'z': {1: 2, 2:4, 3:9, 4:16}})
e = {
'animals': {
'pets': {
'friendly': ['dogs'],
'snobby': ['cats']
}
},
'ready': 0
}
f = {
'animals': {
'pets': {
'friendly': ['dogs', 'rats', 'hamsters'],
'wet': ['goldfish', 'eels']
},
'herbivores': {
'plains': ['elephants']
}
},
'ready': True
}
util.recursive_merge(e, f)
self.assertEqual(e, {
'animals': {
'pets': {
'friendly': ['dogs', 'rats', 'hamsters'],
'snobby': ['cats'],
'wet': ['goldfish', 'eels']
},
'herbivores': {
'plains': ['elephants']
}
},
'ready': True
})