def test_substitution_cycle(self):
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string("""
a = ${b}
b = ${c}
c = ${a}
""")
def test_substitutions_overwrite(self):
config1 = ConfigFactory.parse_string(
"""
a = 123
a = ${?test}
a = 5
"""
)
assert config1["a"] == 5
config2 = ConfigFactory.parse_string(
"""
{
database {
host = "localhost"
port = 8000
url = ${database.host}":"${database.port}
}
database {
host = ${?DB_HOST}
}
database {
host = "other.host.net"
port = 433
}
}
"""
)
assert config2["database"]["host"] == "other.host.net"
assert config2["database"]["port"] == 433
assert config2["database"]["url"] == "other.host.net:433"
def test_fallback_substitutions_overwrite(self):
config1 = ConfigFactory.parse_string("""
a = {
b: 1
c: 2
}
""")
config2 = ConfigFactory.parse_string("""
a.b = 4
a.d = 3
""")
config3 = config1.with_fallback(config2)
assert config3['a'] == {'b': 1, 'c': 2, 'd': 3}
config4 = ConfigFactory.parse_string("""
name: foo
""")
config5 = ConfigFactory.parse_string(u"""
longName: "long "${?name}
""",
resolve=False)
config6 = config4.with_fallback(config5)
assert config6 == {'longName': 'long foo', 'name': 'foo'}
def test_list_substitutions(self):
config = ConfigFactory.parse_string("""
common_modules = [php, python]
host_modules = ${common_modules} [java]
""")
assert config.get('host_modules') == ['php', 'python', 'java']
config2 = ConfigFactory.parse_string("""
common_modules = [php, python]
host_modules = [java] ${common_modules}
""")
assert config2.get('host_modules') == ['java', 'php', 'python']
config3 = ConfigFactory.parse_string("""
common_modules = [php, python]
host_modules = [java] ${common_modules} [perl]
""")
assert config3.get('common_modules') == ['php', 'python']
assert config3.get('host_modules') == ['java', 'php', 'python', 'perl']
config4 = ConfigFactory.parse_string("""
common_modules = [php, python]
host_modules = [java] ${common_modules} [perl]
full_modules = ${host_modules} [c, go]
""")
assert config4.get('common_modules') == ['php', 'python']
assert config4.get('host_modules') == ['java', 'php', 'python', 'perl']
assert config4.get('full_modules') == [
'java', 'php', 'python', 'perl', 'c', 'go'
]
def test_include_list(self):
with tempfile.NamedTemporaryFile('w') as fdin:
fdin.write('[1, 2]')
fdin.flush()
config1 = ConfigFactory.parse_string("""
a: [
include "{tmp_file}"
3
4
]
""".format(tmp_file=fdin.name))
assert config1['a'] == [1, 2, 3, 4]
config2 = ConfigFactory.parse_string("""
a: [
3
4
include "{tmp_file}"
]
""".format(tmp_file=fdin.name))
assert config2['a'] == [3, 4, 1, 2]
config3 = ConfigFactory.parse_string("""
a: [
3
include "{tmp_file}"
4
]
""".format(tmp_file=fdin.name))
assert config3['a'] == [3, 1, 2, 4]
def test_include_dict(self):
expected_res = {'a': 1, 'b': 2, 'c': 3, 'd': 4}
with tempfile.NamedTemporaryFile('w') as fdin:
fdin.write('{a: 1, b: 2}')
fdin.flush()
config1 = ConfigFactory.parse_string("""
a: {{
include "{tmp_file}"
c: 3
d: 4
}}
""".format(tmp_file=fdin.name))
assert config1['a'] == expected_res
config2 = ConfigFactory.parse_string("""
a: {{
c: 3
d: 4
include "{tmp_file}"
}}
""".format(tmp_file=fdin.name))
assert config2['a'] == expected_res
config3 = ConfigFactory.parse_string("""
a: {{
c: 3
include "{tmp_file}"
d: 4
}}
""".format(tmp_file=fdin.name))
assert config3['a'] == expected_res
def test_substitutions_overwrite(self):
config1 = ConfigFactory.parse_string(
"""
a = 123
a = ${?test}
a = 5
"""
)
assert config1['a'] == 5
config2 = ConfigFactory.parse_string(
"""
{
database {
host = "localhost"
port = 8000
url = ${database.host}":"${database.port}
}
database {
host = ${?DB_HOST}
}
database {
host = "other.host.net"
port = 433
}
}
"""
)
assert config2['database']['host'] == 'other.host.net'
assert config2['database']['port'] == 433
assert config2['database']['url'] == 'other.host.net:433'
def test_substitutions_overwrite(self):
config1 = ConfigFactory.parse_string(
"""
a = 123
a = ${?test}
a = 5
"""
)
assert config1['a'] == 5
config2 = ConfigFactory.parse_string(
"""
{
database {
host = "localhost"
port = 8000
url = ${database.host}":"${database.port}
}
database {
host = ${?DB_HOST}
}
database {
host = "other.host.net"
port = 433
}
}
"""
)
assert config2['database']['host'] == 'other.host.net'
assert config2['database']['port'] == 433
assert config2['database']['url'] == 'other.host.net:433'
def test_substitution_cycle(self):
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
a = ${b}
b = ${c}
c = ${a}
""")
def test_string_substitutions(self):
config1 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = ${a.b.c}
f = ${a.b.e}
}
"""
)
assert config1.get('a.b.c') == 'str'
assert config1.get('d') == 'str'
assert config1.get('f') == 'str '
config2 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = test ${a.b.c}
f = test ${a.b.e}
}
"""
)
assert config2.get('a.b.c') == 'str'
assert config2.get('d') == 'test str'
assert config2.get('f') == 'test str '
config3 = ConfigFactory.parse_string(
u"""
{
a: {
b: {
c = str
e = "str "
}
}
d = test ${a.b.c} me
f = test ${a.b.e} me
}
"""
)
assert config3.get('a.b.c') == 'str'
assert config3.get('d') == 'test str me'
assert config3.get('f') == 'test str me'
def test_string_substitutions(self):
config1 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = ${a.b.c}
f = ${a.b.e}
}
"""
)
assert config1.get('a.b.c') == 'str'
assert config1.get('d') == 'str'
assert config1.get('f') == 'str '
config2 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = test ${a.b.c}
f = test ${a.b.e}
}
"""
)
assert config2.get('a.b.c') == 'str'
assert config2.get('d') == 'test str'
assert config2.get('f') == 'test str '
config3 = ConfigFactory.parse_string(
u"""
{
a: {
b: {
c = str
e = "str "
}
}
d = test ${a.b.c} me
f = test ${a.b.e} me
}
"""
)
assert config3.get('a.b.c') == 'str'
assert config3.get('d') == 'test str me'
assert config3.get('f') == 'test str me'
def test_string_substitutions(self):
config1 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = ${a.b.c}
f = ${a.b.e}
}
"""
)
assert config1.get("a.b.c") == "str"
assert config1.get("d") == "str"
assert config1.get("f") == "str "
config2 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = test ${a.b.c}
f = test ${a.b.e}
}
"""
)
assert config2.get("a.b.c") == "str"
assert config2.get("d") == "test str"
assert config2.get("f") == "test str "
config3 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = test ${a.b.c} me
f = test ${a.b.e} me
}
"""
)
assert config3.get("a.b.c") == "str"
assert config3.get("d") == "test str me"
assert config3.get("f") == "test str me"
def test_fallback_self_ref_substitutions_merge(self):
config1 = ConfigFactory.parse_string("""
dict = { x: 1 }
""")
config2 = ConfigFactory.parse_string("""
dict = ${dict} { y: 2 }
""",
resolve=False)
config2 = config2.with_fallback(config1)
assert config2.get("dict") == {'x': 1, 'y': 2}
def test_fallback_self_ref_substitutions_append_plus_equals(self):
config1 = ConfigFactory.parse_string("""
list = [ 1, 2, 3 ]
""")
config2 = ConfigFactory.parse_string("""
list += [ 4, 5, 6 ]
""",
resolve=False)
config2 = config2.with_fallback(config1)
assert config2.get("list") == [1, 2, 3, 4, 5, 6]
def test_fallback_self_ref_substitutions_concat_string(self):
config1 = ConfigFactory.parse_string("""
string = abc
""")
config2 = ConfigFactory.parse_string("""
string = ${string}def
""",
resolve=False)
config2 = config2.with_fallback(config1)
assert config2.get("string") == 'abcdef'
def test_invalid_dict(self):
with pytest.raises(ParseSyntaxException):
ConfigFactory.parse_string("""
a = {
f: 5
g
}
""")
with pytest.raises(ParseSyntaxException):
ConfigFactory.parse_string('a = {g}')
def test_invalid_dict(self):
with pytest.raises(ParseSyntaxException):
ConfigFactory.parse_string(
"""
a = {
f: 5
g
}
""")
with pytest.raises(ParseSyntaxException):
ConfigFactory.parse_string('a = {g}')
def test_fallback_self_ref_substitutions_append_plus_equals(self):
config1 = ConfigFactory.parse_string(
"""
list = [ 1, 2, 3 ]
"""
)
config2 = ConfigFactory.parse_string(
"""
list += [ 4, 5, 6 ]
""",
resolve=False
)
config2 = config2.with_fallback(config1)
assert config2.get("list") == [1, 2, 3, 4, 5, 6]
def test_fallback_self_ref_substitutions_merge(self):
config1 = ConfigFactory.parse_string(
"""
dict = { x: 1 }
"""
)
config2 = ConfigFactory.parse_string(
"""
dict = ${dict} { y: 2 }
""",
resolve=False
)
config2 = config2.with_fallback(config1)
assert config2.get("dict") == {'x': 1, 'y': 2}
def test_fallback_self_ref_substitutions_concat_string(self):
config1 = ConfigFactory.parse_string(
"""
string = abc
"""
)
config2 = ConfigFactory.parse_string(
"""
string = ${string}def
""",
resolve=False
)
config2 = config2.with_fallback(config1)
assert config2.get("string") == 'abcdef'
def test_function_parser(self):
parser = app.FunctionParser()
fn = parser.parse('foo', ConfigFactory.parse_string("""
class-name = "SimpleContext"
context = "test-context-2"
"""))
self.assertIsInstance(fn, models.Function)
self.assertEqual(fn.name, 'foo')
self.assertEqual(fn.default_context.name, 'test-context-2')
self.assertEqual(fn.class_name, 'SimpleContext')
default_fn = parser.parse('foo', ConfigFactory.parse_string("""
class-name = "SimpleContext"
"""))
self.assertEqual(default_fn.default_context.name, 'default')
def test_non_existent_substitution(self):
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = ${non_existent}
"""
)
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = abc ${non_existent}
"""
)
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = ${non_existent} abc
"""
)
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = abc ${non_existent} def
"""
)
def test_non_existent_substitution(self):
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = ${non_existent}
"""
)
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = abc ${non_existent}
"""
)
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = ${non_existent} abc
"""
)
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = abc ${non_existent} def
"""
)
def test_self_merge_ref_substitutions_object(self):
config1 = ConfigFactory.parse_string("""
a : { }
b : 1
c : ${a} { d : [ ${b} ] }
""",
resolve=False)
config2 = ConfigFactory.parse_string("""
e : ${a} {
}
""",
resolve=False)
merged = ConfigTree.merge_configs(config1, config2)
resolved = ConfigParser.resolve_substitutions(merged)
assert resolved.get("c.d") == [1]
def test_bad_concat(self):
ConfigFactory.parse_string('a = 45\n')
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string('a = [4] "4"')
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string('a = "4" [5]')
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string('a = {b: 5} "4"')
def test_bad_concat(self):
ConfigFactory.parse_string('a = 45\n')
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string('a = [4] "4"')
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string('a = "4" [5]')
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string('a = {b: 5} "4"')
def test_int_substitutions(self):
config1 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = 5
}
}
d = ${a.b.c}
}
"""
)
assert config1.get('a.b.c') == 5
assert config1.get('d') == 5
config2 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = 5
}
}
d = test ${a.b.c}
}
"""
)
assert config2.get('a.b.c') == 5
assert config2.get('d') == 'test 5'
config3 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = 5
}
}
d = test ${a.b.c} me
}
"""
)
assert config3.get('a.b.c') == 5
assert config3.get('d') == 'test 5 me'
def test_int_substitutions(self):
config1 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = 5
}
}
d = ${a.b.c}
}
"""
)
assert config1.get('a.b.c') == 5
assert config1.get('d') == 5
config2 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = 5
}
}
d = test ${a.b.c}
}
"""
)
assert config2.get('a.b.c') == 5
assert config2.get('d') == 'test 5'
config3 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = 5
}
}
d = test ${a.b.c} me
}
"""
)
assert config3.get('a.b.c') == 5
assert config3.get('d') == 'test 5 me'
def test_assign_list_strings_with_eol(self):
config = ConfigFactory.parse_string(
"""
a =
[
"a",
"b",
]
b = # test
# test2
[
"c",
"d",]
c =
[
"e",
"f"
]
"""
)
assert config['a'] == ['a', 'b']
assert config['b'] == ['c', 'd']
assert config['c'] == ['e', 'f']
def single_run(policy,
trace,
size=4,
changes={},
name=None,
save=True,
reuse=False,
verbose=False,
readonly=False):
name = name if name else policy
policy = Policy[policy]
trace = Trace[trace]
if 0 < size < 9:
size = trace.typical_caches()[size - 1]
conf_path = caffeine_root + 'simulator{0}src{0}main{0}resources{0}'.format(
os.sep)
conf_file = conf_path + 'application.conf'
if not os.path.exists(output_csvs_path):
os.makedirs(output_csvs_path)
run_simulator = './gradlew simulator:run -x caffeine:compileJava -x caffeine:compileCodeGenJava'
# run_simulator = './gradlew simulator:run'
if os.path.exists(conf_file):
conf = ConfigFactory.parse_file(conf_file)
else:
conf = ConfigFactory.parse_string("""
caffeine {
simulator {
}
}
""")
simulator = conf['caffeine']['simulator']
simulator.put('files.paths',
[resources_path + trace.format() + os.sep + trace.file()])
simulator.put('files.format', trace.value['format'])
simulator.put('maximum-size', size)
simulator.put('policies', [policy.value])
simulator.put('admission', [Admission.ALWAYS.value])
simulator.put('report.format', 'csv')
simulator.put(
'report.output',
output_csvs_path + '{}-{}-{}.csv'.format(trace.name, size, name))
for k, v in changes.items():
simulator.put(k, v)
with open(conf_file, 'w') as f:
f.write(HOCONConverter.to_hocon(conf))
if (not reuse or not os.path.isfile(
simulator['report']['output'])) and not readonly:
call(run_simulator,
shell=True,
cwd=caffeine_root,
stdout=subprocess.DEVNULL if not verbose else None)
with open(simulator['report']['output'], 'r') as csvfile:
reader = csv.DictReader(csvfile)
results = {line['Policy']: float(line['Hit rate']) for line in reader}
if not save:
os.remove(simulator['report']['output'])
return results if len(results) != 1 else results[0]
def test_assign_dict_strings_no_equal_sign_with_eol(self):
config = ConfigFactory.parse_string(
"""
a
{
a: 1,
b: 2,
}
b # test
# test2
{
c: 3,
d: 4,}
c
{
e: 5,
f: 6
}
"""
)
assert config["a"] == {"a": 1, "b": 2}
assert config["b"] == {"c": 3, "d": 4}
assert config["c"] == {"e": 5, "f": 6}
def test_self_ref_substitution_dict_path_hide(self):
config = ConfigFactory.parse_string("""
x = {y: {y: 1}}
x = ${x.y}
""")
assert config.get("x.y") == {'y': 1}
assert set(config.get("x").keys()) == set(['y'])
def test_parse_with_comments(self):
config = ConfigFactory.parse_string(
"""
// comment 1
# comment 2
{
c = test // comment 0
g = 6 test # comment 0
# comment 3
a: { # comment 4
b: test, # comment 5
} # comment 6
t = [1, # comment 7
2, # comment 8
3, # comment 9
]
} # comment 10
// comment 11
// comment 12
"""
)
assert config.get('c') == 'test'
assert config.get('g') == '6 test'
assert config.get('a.b') == 'test'
assert config.get_string('a.b') == 'test'
assert config.get('t') == [1, 2, 3]
def test_assign_list_strings_with_eol(self):
config = ConfigFactory.parse_string(
"""
a =
[
"a",
"b",
]
b = # test
# test2
[
"c",
"d",]
c =
[
"e",
"f"
]
"""
)
assert config['a'] == ['a', 'b']
assert config['b'] == ['c', 'd']
assert config['c'] == ['e', 'f']
def test_self_append_nonexistent_object(self):
config = ConfigFactory.parse_string(
"""
x += {a: 1}
"""
)
assert config.get("x") == {'a': 1}
def test_parse_with_comments(self):
config = ConfigFactory.parse_string(
"""
// comment 1
# comment 2
{
c = test // comment 0
g = 6 test # comment 0
# comment 3
a: { # comment 4
b: test, # comment 5
} # comment 6
t = [1, # comment 7
2, # comment 8
3, # comment 9
]
} # comment 10
// comment 11
// comment 12
"""
)
assert config.get("c") == "test"
assert config.get("g") == "6 test"
assert config.get("a.b") == "test"
assert config.get_string("a.b") == "test"
assert config.get("t") == [1, 2, 3]
def test_missing_config(self):
config = ConfigFactory.parse_string("""
a = 5
""")
# b is not set so show raise an exception
with pytest.raises(ConfigMissingException):
config.get('b')
def test_cascade_optional_substitution(self):
config = ConfigFactory.parse_string("""
num = 3
retries_msg = You have ${num} retries
retries_msg = ${?CUSTOM_MSG}
""")
assert config == {'num': 3, 'retries_msg': 'You have 3 retries'}
def test_list_element_substitution(self):
config = ConfigFactory.parse_string("""
main_language = php
languages = [java, ${main_language}]
""")
assert config.get('languages') == ['java', 'php']
def test_parse_null(self):
config = ConfigFactory.parse_string(
"""
a = null
"""
)
assert config.get('a') is None
def test_assign_dict_strings_no_equal_sign_with_eol(self):
config = ConfigFactory.parse_string(
"""
a
{
a: 1,
b: 2,
}
b # test
# test2
{
c: 3,
d: 4,}
c
{
e: 5,
f: 6
}
"""
)
assert config['a'] == {'a': 1, 'b': 2}
assert config['b'] == {'c': 3, 'd': 4}
assert config['c'] == {'e': 5, 'f': 6}
def test_validation_success(self):
job = WordCountSparkJob()
result = job.validate(
self.sc,
None,
ConfigFactory.parse_string('input.strings = ["a", "a", "b"]'))
self.assertEqual(result, ['a', 'a', 'b'])
def test_self_append_nonexistent_array(self):
config = ConfigFactory.parse_string(
"""
x += [1,2]
"""
)
assert config.get("x") == [1, 2]
def test_assign_list_numbers_with_eol(self):
config = ConfigFactory.parse_string(
"""
a =
[
1,
2,
]
b = # test
# test2
[
3,
4,]
c =
[
5,
6
]
"""
)
assert config['a'] == [1, 2]
assert config['b'] == [3, 4]
assert config['c'] == [5, 6]
def test_parse_null(self):
config = ConfigFactory.parse_string(
"""
a = null
"""
)
assert config.get('a') is None
def test_assign_dict_strings_no_equal_sign_with_eol(self):
config = ConfigFactory.parse_string(
"""
a
{
a: 1,
b: 2,
}
b # test
# test2
{
c: 3,
d: 4,}
c
{
e: 5,
f: 6
}
"""
)
assert config['a'] == {'a': 1, 'b': 2}
assert config['b'] == {'c': 3, 'd': 4}
assert config['c'] == {'e': 5, 'f': 6}
def test_parse_simple_value(self):
config = ConfigFactory.parse_string(
"""t = {
c = 5
"d" = true
e.y = {
f: 7
g: "hey dude!"
h: hey man!
i = \"\"\"
"first line"
"second" line
\"\"\"
}
j = [1, 2, 3]
u = 192.168.1.3/32
}
"""
)
assert config.get_string('t.c') == '5'
assert config.get_int('t.c') == 5
assert config.get('t.e.y.f') == 7
assert config.get('t.e.y.g') == 'hey dude!'
assert config.get('t.e.y.h') == 'hey man!'
assert [l.strip() for l in config.get('t.e.y.i').split('\n')] == ['', '"first line"', '"second" line', '']
assert config.get_bool('t.d') is True
assert config.get_int('t.e.y.f') == 7
assert config.get('t.j') == [1, 2, 3]
assert config.get('t.u') == '192.168.1.3/32'
def test_parse_simple_value(self):
config = ConfigFactory.parse_string(
"""t = {
c = 5
"d" = true
e.y = {
f: 7
g: "hey dude!"
h: hey man!
i = \"\"\"
"first line"
"second" line
\"\"\"
}
j = [1, 2, 3]
u = 192.168.1.3/32
}
"""
)
assert config.get_string('t.c') == '5'
assert config.get_int('t.c') == 5
assert config.get('t.e.y.f') == 7
assert config.get('t.e.y.g') == 'hey dude!'
assert config.get('t.e.y.h') == 'hey man!'
assert [l.strip() for l in config.get('t.e.y.i').split('\n')] == ['', '"first line"', '"second" line', '']
assert config.get_bool('t.d') is True
assert config.get_int('t.e.y.f') == 7
assert config.get('t.j') == [1, 2, 3]
assert config.get('t.u') == '192.168.1.3/32'
def test_parse_simple_value(self):
config = ConfigFactory.parse_string(
"""t = {
c = 5
"d" = true
e.y = {
f: 7
g: "hey dude!"
h: hey man!
i = \"\"\"
"first line"
"second" line
\"\"\"
}
j = [1, 2, 3]
u = 192.168.1.3/32
}
"""
)
assert config.get_string("t.c") == "5"
assert config.get_int("t.c") == 5
assert config.get("t.e.y.f") == 7
assert config.get("t.e.y.g") == "hey dude!"
assert config.get("t.e.y.h") == "hey man!"
assert [l.strip() for l in config.get("t.e.y.i").split("\n")] == ["", '"first line"', '"second" line', ""]
assert config.get_bool("t.d") is True
assert config.get_int("t.e.y.f") == 7
assert config.get("t.j") == [1, 2, 3]
assert config.get("t.u") == "192.168.1.3/32"
def test_parse_with_comments(self):
config = ConfigFactory.parse_string(
"""
// comment 1
# comment 2
{
c = test // comment 0
g = 6 test # comment 0
# comment 3
a: { # comment 4
b: test, # comment 5
} # comment 6
t = [1, # comment 7
2, # comment 8
3, # comment 9
]
} # comment 10
// comment 11
// comment 12
"""
)
assert config.get('c') == 'test'
assert config.get('g') == '6 test'
assert config.get('a.b') == 'test'
assert config.get_string('a.b') == 'test'
assert config.get('t') == [1, 2, 3]
def test_assign_list_strings_with_eol(self):
config = ConfigFactory.parse_string(
"""
a =
[
"a",
"b",
]
b = # test
# test2
[
"c",
"d",]
c =
[
"e",
"f"
]
"""
)
assert config["a"] == ["a", "b"]
assert config["b"] == ["c", "d"]
assert config["c"] == ["e", "f"]
def test_dict_merge(self):
config = ConfigFactory.parse_string(
"""
a {
d {
g.h.j.u: 5
g {
h.d: 4
}
g.h.k: f d
}
h.i.m = 7
h.i {
d: 5
}
h.i {
e:65
}
}
"""
)
expected_result = {
"a": {"d": {"g": {"h": {"j": {"u": 5}, "d": 4, "k": "f d"}}}, "h": {"i": {"m": 7, "d": 5, "e": 65}}}
}
assert expected_result == config
def convert(input_file=None, output_file=None, format='json'):
"""Convert to json, properties or yaml
:param format: json, properties or yaml
:type format: basestring
:return: json, properties or yaml string representation
"""
if input_file is None:
content = sys.stdin.read()
config = ConfigFactory.parse_string(content)
else:
config = ConfigFactory.parse_file(input_file)
if format.lower() == 'json':
res = HOCONConverter.to_json(config)
elif format.lower() == 'properties':
res = HOCONConverter.to_properties(config)
elif format.lower() == 'yaml':
res = HOCONConverter.to_yaml(config)
else:
raise Exception("Format must be 'json', 'properties' or 'yaml'")
if output_file is None:
print(res)
else:
with open(output_file, "w") as fd:
fd.write(res)
def convert_from_file(cls,
input_file=None,
output_file=None,
output_format='json',
indent=2,
compact=False):
"""Convert to json, properties or yaml
:param input_file: input file, if not specified stdin
:param output_file: output file, if not specified stdout
:param output_format: json, properties or yaml
:return: json, properties or yaml string representation
"""
if input_file is None:
content = sys.stdin.read()
config = ConfigFactory.parse_string(content)
else:
config = ConfigFactory.parse_file(input_file)
res = cls.convert(config, output_format, indent, compact)
if output_file is None:
print(res)
else:
with open(output_file, "w") as fd:
fd.write(res)
def test_assign_list_numbers_with_eol(self):
config = ConfigFactory.parse_string(
"""
a =
[
1,
2,
]
b = # test
# test2
[
3,
4,]
c =
[
5,
6
]
"""
)
assert config['a'] == [1, 2]
assert config['b'] == [3, 4]
assert config['c'] == [5, 6]
def test_custom_settings_no_resolver_config(self):
custom_settings = ConfigFactory.parse_string(
strip_margin("""|dummy = foo
|"""))
result = resolver.resolver_chain(custom_settings, False)
expected_result = [stdin_resolver, uri_resolver, bintray_resolver]
self.assertEqual(expected_result, result)
def load_v1(args):
log = logging.getLogger(__name__)
log.info('Retrieving bundle...')
custom_settings = args.custom_settings
resolve_cache_dir = args.resolve_cache_dir
bundle_name, bundle_file = resolver.resolve_bundle(custom_settings, resolve_cache_dir, args.bundle)
configuration_name, configuration_file = (None, None)
if args.configuration is not None:
log.info('Retrieving configuration...')
configuration_name, configuration_file = resolver.resolve_bundle(custom_settings, resolve_cache_dir, args.configuration)
bundle_conf = ConfigFactory.parse_string(bundle_utils.conf(bundle_file))
overlay_bundle_conf = None if configuration_file is None else \
ConfigFactory.parse_string(bundle_utils.conf(configuration_file))
with_bundle_configurations = partial(apply_to_configurations, bundle_conf, overlay_bundle_conf)
url = conduct_url.url('bundles', args)
files = get_payload(bundle_name, bundle_file, with_bundle_configurations)
if configuration_file is not None:
files.append(('configuration', (configuration_name, open(configuration_file, 'rb'))))
log.info('Loading bundle to ConductR...')
response = requests.post(url, files=files, timeout=LOAD_HTTP_TIMEOUT)
validation.raise_for_status_inc_3xx(response)
if log.is_verbose_enabled():
log.verbose(validation.pretty_json(response.text))
response_json = json.loads(response.text)
bundle_id = response_json['bundleId'] if args.long_ids else bundle_utils.short_id(response_json['bundleId'])
if not args.no_wait:
bundle_installation.wait_for_installation(response_json['bundleId'], args)
log.info('Bundle loaded.')
log.info('Start bundle with: conduct run{} {}'.format(args.cli_parameters, bundle_id))
log.info('Unload bundle with: conduct unload{} {}'.format(args.cli_parameters, bundle_id))
log.info('Print ConductR info with: conduct info{}'.format(args.cli_parameters))
if not log.is_info_enabled() and log.is_quiet_enabled():
log.quiet(response_json['bundleId'])
return True
def test_one_line_quote_escape(self):
config = ConfigFactory.parse_string(
"""
test: "abc\n\n"
"""
)
assert config['test'] == 'abc\n\n'
def test_self_ref_substitution_dict_recurse(self):
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
x = ${x}
"""
)
