def test_substitution_cycle(self): with pytest.raises(ConfigSubstitutionException): ConfigFactory.parse_string(""" a = ${b} b = ${c} c = ${a} """)
def test_substitutions_overwrite(self): config1 = ConfigFactory.parse_string( """ a = 123 a = ${?test} a = 5 """ ) assert config1["a"] == 5 config2 = ConfigFactory.parse_string( """ { database { host = "localhost" port = 8000 url = ${database.host}":"${database.port} } database { host = ${?DB_HOST} } database { host = "other.host.net" port = 433 } } """ ) assert config2["database"]["host"] == "other.host.net" assert config2["database"]["port"] == 433 assert config2["database"]["url"] == "other.host.net:433"
def test_fallback_substitutions_overwrite(self): config1 = ConfigFactory.parse_string(""" a = { b: 1 c: 2 } """) config2 = ConfigFactory.parse_string(""" a.b = 4 a.d = 3 """) config3 = config1.with_fallback(config2) assert config3['a'] == {'b': 1, 'c': 2, 'd': 3} config4 = ConfigFactory.parse_string(""" name: foo """) config5 = ConfigFactory.parse_string(u""" longName: "long "${?name} """, resolve=False) config6 = config4.with_fallback(config5) assert config6 == {'longName': 'long foo', 'name': 'foo'}
def test_list_substitutions(self): config = ConfigFactory.parse_string(""" common_modules = [php, python] host_modules = ${common_modules} [java] """) assert config.get('host_modules') == ['php', 'python', 'java'] config2 = ConfigFactory.parse_string(""" common_modules = [php, python] host_modules = [java] ${common_modules} """) assert config2.get('host_modules') == ['java', 'php', 'python'] config3 = ConfigFactory.parse_string(""" common_modules = [php, python] host_modules = [java] ${common_modules} [perl] """) assert config3.get('common_modules') == ['php', 'python'] assert config3.get('host_modules') == ['java', 'php', 'python', 'perl'] config4 = ConfigFactory.parse_string(""" common_modules = [php, python] host_modules = [java] ${common_modules} [perl] full_modules = ${host_modules} [c, go] """) assert config4.get('common_modules') == ['php', 'python'] assert config4.get('host_modules') == ['java', 'php', 'python', 'perl'] assert config4.get('full_modules') == [ 'java', 'php', 'python', 'perl', 'c', 'go' ]
def test_include_list(self): with tempfile.NamedTemporaryFile('w') as fdin: fdin.write('[1, 2]') fdin.flush() config1 = ConfigFactory.parse_string(""" a: [ include "{tmp_file}" 3 4 ] """.format(tmp_file=fdin.name)) assert config1['a'] == [1, 2, 3, 4] config2 = ConfigFactory.parse_string(""" a: [ 3 4 include "{tmp_file}" ] """.format(tmp_file=fdin.name)) assert config2['a'] == [3, 4, 1, 2] config3 = ConfigFactory.parse_string(""" a: [ 3 include "{tmp_file}" 4 ] """.format(tmp_file=fdin.name)) assert config3['a'] == [3, 1, 2, 4]
def test_include_dict(self): expected_res = {'a': 1, 'b': 2, 'c': 3, 'd': 4} with tempfile.NamedTemporaryFile('w') as fdin: fdin.write('{a: 1, b: 2}') fdin.flush() config1 = ConfigFactory.parse_string(""" a: {{ include "{tmp_file}" c: 3 d: 4 }} """.format(tmp_file=fdin.name)) assert config1['a'] == expected_res config2 = ConfigFactory.parse_string(""" a: {{ c: 3 d: 4 include "{tmp_file}" }} """.format(tmp_file=fdin.name)) assert config2['a'] == expected_res config3 = ConfigFactory.parse_string(""" a: {{ c: 3 include "{tmp_file}" d: 4 }} """.format(tmp_file=fdin.name)) assert config3['a'] == expected_res
def test_substitutions_overwrite(self): config1 = ConfigFactory.parse_string( """ a = 123 a = ${?test} a = 5 """ ) assert config1['a'] == 5 config2 = ConfigFactory.parse_string( """ { database { host = "localhost" port = 8000 url = ${database.host}":"${database.port} } database { host = ${?DB_HOST} } database { host = "other.host.net" port = 433 } } """ ) assert config2['database']['host'] == 'other.host.net' assert config2['database']['port'] == 433 assert config2['database']['url'] == 'other.host.net:433'
def test_substitution_cycle(self): with pytest.raises(ConfigSubstitutionException): ConfigFactory.parse_string( """ a = ${b} b = ${c} c = ${a} """)
def test_string_substitutions(self): config1 = ConfigFactory.parse_string( """ { a: { b: { c = str e = "str " } } d = ${a.b.c} f = ${a.b.e} } """ ) assert config1.get('a.b.c') == 'str' assert config1.get('d') == 'str' assert config1.get('f') == 'str ' config2 = ConfigFactory.parse_string( """ { a: { b: { c = str e = "str " } } d = test ${a.b.c} f = test ${a.b.e} } """ ) assert config2.get('a.b.c') == 'str' assert config2.get('d') == 'test str' assert config2.get('f') == 'test str ' config3 = ConfigFactory.parse_string( u""" { a: { b: { c = str e = "str " } } d = test ${a.b.c} me f = test ${a.b.e} me } """ ) assert config3.get('a.b.c') == 'str' assert config3.get('d') == 'test str me' assert config3.get('f') == 'test str me'
def test_string_substitutions(self): config1 = ConfigFactory.parse_string( """ { a: { b: { c = str e = "str " } } d = ${a.b.c} f = ${a.b.e} } """ ) assert config1.get("a.b.c") == "str" assert config1.get("d") == "str" assert config1.get("f") == "str " config2 = ConfigFactory.parse_string( """ { a: { b: { c = str e = "str " } } d = test ${a.b.c} f = test ${a.b.e} } """ ) assert config2.get("a.b.c") == "str" assert config2.get("d") == "test str" assert config2.get("f") == "test str " config3 = ConfigFactory.parse_string( """ { a: { b: { c = str e = "str " } } d = test ${a.b.c} me f = test ${a.b.e} me } """ ) assert config3.get("a.b.c") == "str" assert config3.get("d") == "test str me" assert config3.get("f") == "test str me"
def test_fallback_self_ref_substitutions_merge(self): config1 = ConfigFactory.parse_string(""" dict = { x: 1 } """) config2 = ConfigFactory.parse_string(""" dict = ${dict} { y: 2 } """, resolve=False) config2 = config2.with_fallback(config1) assert config2.get("dict") == {'x': 1, 'y': 2}
def test_fallback_self_ref_substitutions_append_plus_equals(self): config1 = ConfigFactory.parse_string(""" list = [ 1, 2, 3 ] """) config2 = ConfigFactory.parse_string(""" list += [ 4, 5, 6 ] """, resolve=False) config2 = config2.with_fallback(config1) assert config2.get("list") == [1, 2, 3, 4, 5, 6]
def test_fallback_self_ref_substitutions_concat_string(self): config1 = ConfigFactory.parse_string(""" string = abc """) config2 = ConfigFactory.parse_string(""" string = ${string}def """, resolve=False) config2 = config2.with_fallback(config1) assert config2.get("string") == 'abcdef'
def test_invalid_dict(self): with pytest.raises(ParseSyntaxException): ConfigFactory.parse_string(""" a = { f: 5 g } """) with pytest.raises(ParseSyntaxException): ConfigFactory.parse_string('a = {g}')
def test_invalid_dict(self): with pytest.raises(ParseSyntaxException): ConfigFactory.parse_string( """ a = { f: 5 g } """) with pytest.raises(ParseSyntaxException): ConfigFactory.parse_string('a = {g}')
def test_fallback_self_ref_substitutions_append_plus_equals(self): config1 = ConfigFactory.parse_string( """ list = [ 1, 2, 3 ] """ ) config2 = ConfigFactory.parse_string( """ list += [ 4, 5, 6 ] """, resolve=False ) config2 = config2.with_fallback(config1) assert config2.get("list") == [1, 2, 3, 4, 5, 6]
def test_fallback_self_ref_substitutions_merge(self): config1 = ConfigFactory.parse_string( """ dict = { x: 1 } """ ) config2 = ConfigFactory.parse_string( """ dict = ${dict} { y: 2 } """, resolve=False ) config2 = config2.with_fallback(config1) assert config2.get("dict") == {'x': 1, 'y': 2}
def test_fallback_self_ref_substitutions_concat_string(self): config1 = ConfigFactory.parse_string( """ string = abc """ ) config2 = ConfigFactory.parse_string( """ string = ${string}def """, resolve=False ) config2 = config2.with_fallback(config1) assert config2.get("string") == 'abcdef'
def test_function_parser(self): parser = app.FunctionParser() fn = parser.parse('foo', ConfigFactory.parse_string(""" class-name = "SimpleContext" context = "test-context-2" """)) self.assertIsInstance(fn, models.Function) self.assertEqual(fn.name, 'foo') self.assertEqual(fn.default_context.name, 'test-context-2') self.assertEqual(fn.class_name, 'SimpleContext') default_fn = parser.parse('foo', ConfigFactory.parse_string(""" class-name = "SimpleContext" """)) self.assertEqual(default_fn.default_context.name, 'default')
def test_non_existent_substitution(self): with pytest.raises(ConfigSubstitutionException): ConfigFactory.parse_string( """ common_modules = ${non_existent} """ ) with pytest.raises(ConfigSubstitutionException): ConfigFactory.parse_string( """ common_modules = abc ${non_existent} """ ) with pytest.raises(ConfigSubstitutionException): ConfigFactory.parse_string( """ common_modules = ${non_existent} abc """ ) with pytest.raises(ConfigSubstitutionException): ConfigFactory.parse_string( """ common_modules = abc ${non_existent} def """ )
def test_self_merge_ref_substitutions_object(self): config1 = ConfigFactory.parse_string(""" a : { } b : 1 c : ${a} { d : [ ${b} ] } """, resolve=False) config2 = ConfigFactory.parse_string(""" e : ${a} { } """, resolve=False) merged = ConfigTree.merge_configs(config1, config2) resolved = ConfigParser.resolve_substitutions(merged) assert resolved.get("c.d") == [1]
def test_bad_concat(self): ConfigFactory.parse_string('a = 45\n') with pytest.raises(ConfigWrongTypeException): ConfigFactory.parse_string('a = [4] "4"') with pytest.raises(ConfigWrongTypeException): ConfigFactory.parse_string('a = "4" [5]') with pytest.raises(ConfigWrongTypeException): ConfigFactory.parse_string('a = {b: 5} "4"')
def test_int_substitutions(self): config1 = ConfigFactory.parse_string( """ { a: { b: { c = 5 } } d = ${a.b.c} } """ ) assert config1.get('a.b.c') == 5 assert config1.get('d') == 5 config2 = ConfigFactory.parse_string( """ { a: { b: { c = 5 } } d = test ${a.b.c} } """ ) assert config2.get('a.b.c') == 5 assert config2.get('d') == 'test 5' config3 = ConfigFactory.parse_string( """ { a: { b: { c = 5 } } d = test ${a.b.c} me } """ ) assert config3.get('a.b.c') == 5 assert config3.get('d') == 'test 5 me'
def test_assign_list_strings_with_eol(self): config = ConfigFactory.parse_string( """ a = [ "a", "b", ] b = # test # test2 [ "c", "d",] c = [ "e", "f" ] """ ) assert config['a'] == ['a', 'b'] assert config['b'] == ['c', 'd'] assert config['c'] == ['e', 'f']
def single_run(policy, trace, size=4, changes={}, name=None, save=True, reuse=False, verbose=False, readonly=False): name = name if name else policy policy = Policy[policy] trace = Trace[trace] if 0 < size < 9: size = trace.typical_caches()[size - 1] conf_path = caffeine_root + 'simulator{0}src{0}main{0}resources{0}'.format( os.sep) conf_file = conf_path + 'application.conf' if not os.path.exists(output_csvs_path): os.makedirs(output_csvs_path) run_simulator = './gradlew simulator:run -x caffeine:compileJava -x caffeine:compileCodeGenJava' # run_simulator = './gradlew simulator:run' if os.path.exists(conf_file): conf = ConfigFactory.parse_file(conf_file) else: conf = ConfigFactory.parse_string(""" caffeine { simulator { } } """) simulator = conf['caffeine']['simulator'] simulator.put('files.paths', [resources_path + trace.format() + os.sep + trace.file()]) simulator.put('files.format', trace.value['format']) simulator.put('maximum-size', size) simulator.put('policies', [policy.value]) simulator.put('admission', [Admission.ALWAYS.value]) simulator.put('report.format', 'csv') simulator.put( 'report.output', output_csvs_path + '{}-{}-{}.csv'.format(trace.name, size, name)) for k, v in changes.items(): simulator.put(k, v) with open(conf_file, 'w') as f: f.write(HOCONConverter.to_hocon(conf)) if (not reuse or not os.path.isfile( simulator['report']['output'])) and not readonly: call(run_simulator, shell=True, cwd=caffeine_root, stdout=subprocess.DEVNULL if not verbose else None) with open(simulator['report']['output'], 'r') as csvfile: reader = csv.DictReader(csvfile) results = {line['Policy']: float(line['Hit rate']) for line in reader} if not save: os.remove(simulator['report']['output']) return results if len(results) != 1 else results[0]
def test_assign_dict_strings_no_equal_sign_with_eol(self): config = ConfigFactory.parse_string( """ a { a: 1, b: 2, } b # test # test2 { c: 3, d: 4,} c { e: 5, f: 6 } """ ) assert config["a"] == {"a": 1, "b": 2} assert config["b"] == {"c": 3, "d": 4} assert config["c"] == {"e": 5, "f": 6}
def test_self_ref_substitution_dict_path_hide(self): config = ConfigFactory.parse_string(""" x = {y: {y: 1}} x = ${x.y} """) assert config.get("x.y") == {'y': 1} assert set(config.get("x").keys()) == set(['y'])
def test_parse_with_comments(self): config = ConfigFactory.parse_string( """ // comment 1 # comment 2 { c = test // comment 0 g = 6 test # comment 0 # comment 3 a: { # comment 4 b: test, # comment 5 } # comment 6 t = [1, # comment 7 2, # comment 8 3, # comment 9 ] } # comment 10 // comment 11 // comment 12 """ ) assert config.get('c') == 'test' assert config.get('g') == '6 test' assert config.get('a.b') == 'test' assert config.get_string('a.b') == 'test' assert config.get('t') == [1, 2, 3]
def test_self_append_nonexistent_object(self): config = ConfigFactory.parse_string( """ x += {a: 1} """ ) assert config.get("x") == {'a': 1}
def test_parse_with_comments(self): config = ConfigFactory.parse_string( """ // comment 1 # comment 2 { c = test // comment 0 g = 6 test # comment 0 # comment 3 a: { # comment 4 b: test, # comment 5 } # comment 6 t = [1, # comment 7 2, # comment 8 3, # comment 9 ] } # comment 10 // comment 11 // comment 12 """ ) assert config.get("c") == "test" assert config.get("g") == "6 test" assert config.get("a.b") == "test" assert config.get_string("a.b") == "test" assert config.get("t") == [1, 2, 3]
def test_missing_config(self): config = ConfigFactory.parse_string(""" a = 5 """) # b is not set so show raise an exception with pytest.raises(ConfigMissingException): config.get('b')
def test_cascade_optional_substitution(self): config = ConfigFactory.parse_string(""" num = 3 retries_msg = You have ${num} retries retries_msg = ${?CUSTOM_MSG} """) assert config == {'num': 3, 'retries_msg': 'You have 3 retries'}
def test_list_element_substitution(self): config = ConfigFactory.parse_string(""" main_language = php languages = [java, ${main_language}] """) assert config.get('languages') == ['java', 'php']
def test_parse_null(self): config = ConfigFactory.parse_string( """ a = null """ ) assert config.get('a') is None
def test_assign_dict_strings_no_equal_sign_with_eol(self): config = ConfigFactory.parse_string( """ a { a: 1, b: 2, } b # test # test2 { c: 3, d: 4,} c { e: 5, f: 6 } """ ) assert config['a'] == {'a': 1, 'b': 2} assert config['b'] == {'c': 3, 'd': 4} assert config['c'] == {'e': 5, 'f': 6}
def test_validation_success(self): job = WordCountSparkJob() result = job.validate( self.sc, None, ConfigFactory.parse_string('input.strings = ["a", "a", "b"]')) self.assertEqual(result, ['a', 'a', 'b'])
def test_self_append_nonexistent_array(self): config = ConfigFactory.parse_string( """ x += [1,2] """ ) assert config.get("x") == [1, 2]
def test_assign_list_numbers_with_eol(self): config = ConfigFactory.parse_string( """ a = [ 1, 2, ] b = # test # test2 [ 3, 4,] c = [ 5, 6 ] """ ) assert config['a'] == [1, 2] assert config['b'] == [3, 4] assert config['c'] == [5, 6]
def test_parse_simple_value(self): config = ConfigFactory.parse_string( """t = { c = 5 "d" = true e.y = { f: 7 g: "hey dude!" h: hey man! i = \"\"\" "first line" "second" line \"\"\" } j = [1, 2, 3] u = 192.168.1.3/32 } """ ) assert config.get_string('t.c') == '5' assert config.get_int('t.c') == 5 assert config.get('t.e.y.f') == 7 assert config.get('t.e.y.g') == 'hey dude!' assert config.get('t.e.y.h') == 'hey man!' assert [l.strip() for l in config.get('t.e.y.i').split('\n')] == ['', '"first line"', '"second" line', ''] assert config.get_bool('t.d') is True assert config.get_int('t.e.y.f') == 7 assert config.get('t.j') == [1, 2, 3] assert config.get('t.u') == '192.168.1.3/32'
def test_parse_simple_value(self): config = ConfigFactory.parse_string( """t = { c = 5 "d" = true e.y = { f: 7 g: "hey dude!" h: hey man! i = \"\"\" "first line" "second" line \"\"\" } j = [1, 2, 3] u = 192.168.1.3/32 } """ ) assert config.get_string("t.c") == "5" assert config.get_int("t.c") == 5 assert config.get("t.e.y.f") == 7 assert config.get("t.e.y.g") == "hey dude!" assert config.get("t.e.y.h") == "hey man!" assert [l.strip() for l in config.get("t.e.y.i").split("\n")] == ["", '"first line"', '"second" line', ""] assert config.get_bool("t.d") is True assert config.get_int("t.e.y.f") == 7 assert config.get("t.j") == [1, 2, 3] assert config.get("t.u") == "192.168.1.3/32"
def test_assign_list_strings_with_eol(self): config = ConfigFactory.parse_string( """ a = [ "a", "b", ] b = # test # test2 [ "c", "d",] c = [ "e", "f" ] """ ) assert config["a"] == ["a", "b"] assert config["b"] == ["c", "d"] assert config["c"] == ["e", "f"]
def test_dict_merge(self): config = ConfigFactory.parse_string( """ a { d { g.h.j.u: 5 g { h.d: 4 } g.h.k: f d } h.i.m = 7 h.i { d: 5 } h.i { e:65 } } """ ) expected_result = { "a": {"d": {"g": {"h": {"j": {"u": 5}, "d": 4, "k": "f d"}}}, "h": {"i": {"m": 7, "d": 5, "e": 65}}} } assert expected_result == config
def convert(input_file=None, output_file=None, format='json'): """Convert to json, properties or yaml :param format: json, properties or yaml :type format: basestring :return: json, properties or yaml string representation """ if input_file is None: content = sys.stdin.read() config = ConfigFactory.parse_string(content) else: config = ConfigFactory.parse_file(input_file) if format.lower() == 'json': res = HOCONConverter.to_json(config) elif format.lower() == 'properties': res = HOCONConverter.to_properties(config) elif format.lower() == 'yaml': res = HOCONConverter.to_yaml(config) else: raise Exception("Format must be 'json', 'properties' or 'yaml'") if output_file is None: print(res) else: with open(output_file, "w") as fd: fd.write(res)
def convert_from_file(cls, input_file=None, output_file=None, output_format='json', indent=2, compact=False): """Convert to json, properties or yaml :param input_file: input file, if not specified stdin :param output_file: output file, if not specified stdout :param output_format: json, properties or yaml :return: json, properties or yaml string representation """ if input_file is None: content = sys.stdin.read() config = ConfigFactory.parse_string(content) else: config = ConfigFactory.parse_file(input_file) res = cls.convert(config, output_format, indent, compact) if output_file is None: print(res) else: with open(output_file, "w") as fd: fd.write(res)
def test_custom_settings_no_resolver_config(self): custom_settings = ConfigFactory.parse_string( strip_margin("""|dummy = foo |""")) result = resolver.resolver_chain(custom_settings, False) expected_result = [stdin_resolver, uri_resolver, bintray_resolver] self.assertEqual(expected_result, result)
def load_v1(args): log = logging.getLogger(__name__) log.info('Retrieving bundle...') custom_settings = args.custom_settings resolve_cache_dir = args.resolve_cache_dir bundle_name, bundle_file = resolver.resolve_bundle(custom_settings, resolve_cache_dir, args.bundle) configuration_name, configuration_file = (None, None) if args.configuration is not None: log.info('Retrieving configuration...') configuration_name, configuration_file = resolver.resolve_bundle(custom_settings, resolve_cache_dir, args.configuration) bundle_conf = ConfigFactory.parse_string(bundle_utils.conf(bundle_file)) overlay_bundle_conf = None if configuration_file is None else \ ConfigFactory.parse_string(bundle_utils.conf(configuration_file)) with_bundle_configurations = partial(apply_to_configurations, bundle_conf, overlay_bundle_conf) url = conduct_url.url('bundles', args) files = get_payload(bundle_name, bundle_file, with_bundle_configurations) if configuration_file is not None: files.append(('configuration', (configuration_name, open(configuration_file, 'rb')))) log.info('Loading bundle to ConductR...') response = requests.post(url, files=files, timeout=LOAD_HTTP_TIMEOUT) validation.raise_for_status_inc_3xx(response) if log.is_verbose_enabled(): log.verbose(validation.pretty_json(response.text)) response_json = json.loads(response.text) bundle_id = response_json['bundleId'] if args.long_ids else bundle_utils.short_id(response_json['bundleId']) if not args.no_wait: bundle_installation.wait_for_installation(response_json['bundleId'], args) log.info('Bundle loaded.') log.info('Start bundle with: conduct run{} {}'.format(args.cli_parameters, bundle_id)) log.info('Unload bundle with: conduct unload{} {}'.format(args.cli_parameters, bundle_id)) log.info('Print ConductR info with: conduct info{}'.format(args.cli_parameters)) if not log.is_info_enabled() and log.is_quiet_enabled(): log.quiet(response_json['bundleId']) return True
def test_one_line_quote_escape(self): config = ConfigFactory.parse_string( """ test: "abc\n\n" """ ) assert config['test'] == 'abc\n\n'
def test_self_ref_substitution_dict_recurse(self): with pytest.raises(ConfigSubstitutionException): ConfigFactory.parse_string( """ x = ${x} """ )