def _parse_pipfile(self, contents):
# If any outline tables are present...
if ("[packages." in contents) or ("[dev-packages." in contents):
data = toml.loads(contents)
# Convert all outline tables to inline tables.
for section in ("packages", "dev-packages"):
for package in data.get(section, {}):
# Convert things to inline tables — fancy :)
if hasattr(data[section][package], "keys"):
_data = data[section][package]
data[section][package] = toml._get_empty_inline_table(dict)
data[section][package].update(_data)
# We lose comments here, but it's for the best.)
try:
return contoml.loads(toml.dumps(data, preserve=True))
except RuntimeError:
return toml.loads(toml.dumps(data, preserve=True))
else:
# Fallback to toml parser, for large files.
try:
return contoml.loads(contents)
except Exception:
return toml.loads(contents)
def _parse_pipfile(self, contents):
# If any outline tables are present...
if ("[packages." in contents) or ("[dev-packages." in contents):
data = toml.loads(contents)
# Convert all outline tables to inline tables.
for section in ("packages", "dev-packages"):
for package in data.get(section, {}):
# Convert things to inline tables — fancy :)
if hasattr(data[section][package], "keys"):
_data = data[section][package]
data[section][package] = toml.TomlDecoder(
).get_empty_inline_table()
data[section][package].update(_data)
toml_encoder = toml.TomlEncoder(preserve=True)
# We lose comments here, but it's for the best.)
try:
return contoml.loads(toml.dumps(data, encoder=toml_encoder))
except RuntimeError:
return toml.loads(toml.dumps(data, encoder=toml_encoder))
else:
# Fallback to toml parser, for large files.
try:
return contoml.loads(contents)
except Exception:
return toml.loads(contents)
def parsed_pipfile(self):
# Open the pipfile, read it into memory.
with open(self.pipfile_location) as f:
contents = f.read()
# If any outline tables are present...
if ('[packages.' in contents) or ('[dev-packages.' in contents):
data = toml.loads(contents)
# Convert all outline tables to inline tables.
for section in ('packages', 'dev-packages'):
for package in data.get(section, {}):
# Convert things to inline tables — fancy :)
if hasattr(data[section][package], 'keys'):
_data = data[section][package]
data[section][package] = toml._get_empty_inline_table(
dict)
data[section][package].update(_data)
# We lose comments here, but it's for the best.)
return contoml.loads(toml.dumps(data, preserve=True))
else:
return contoml.loads(contents)
def parsed_pipfile(self):
# Open the pipfile, read it into memory.
with open(self.pipfile_location) as f:
contents = f.read()
# If any outline tables are present...
if ('[packages.' in contents) or ('[dev-packages.' in contents):
data = toml.loads(contents)
# Convert all outline tables to inline tables.
for section in ('packages', 'dev-packages'):
for package in data.get(section, {}):
# Convert things to inline tables — fancy :)
if hasattr(data[section][package], 'keys'):
_data = data[section][package]
data[section][package] = toml._get_empty_inline_table(dict)
data[section][package].update(_data)
# We lose comments here, but it's for the best.)
try:
return contoml.loads(toml.dumps(data, preserve=True))
except RuntimeError:
return toml.loads(toml.dumps(data, preserve=True))
else:
# Fallback to toml parser, for large files.
try:
return contoml.loads(contents)
except Exception:
return toml.loads(contents)
def _parse_pipfile(self, contents):
# If any outline tables are present...
if ('[packages.' in contents) or ('[dev-packages.' in contents):
data = toml.loads(contents)
# Convert all outline tables to inline tables.
for section in ('packages', 'dev-packages'):
for package in data.get(section, {}):
# Convert things to inline tables — fancy :)
if hasattr(data[section][package], 'keys'):
_data = data[section][package]
data[section][package] = toml._get_empty_inline_table(
dict)
data[section][package].update(_data)
# We lose comments here, but it's for the best.)
try:
return contoml.loads(toml.dumps(data, preserve=True))
except RuntimeError:
return toml.loads(toml.dumps(data, preserve=True))
else:
# Fallback to toml parser, for large files.
try:
return contoml.loads(contents)
except Exception:
return toml.loads(contents)
def test_fails_to_parse_bad_escape_characters():
toml = r"""
invalid-escape = r"This string has a bad \a escape character."
"""
try:
contoml.loads(toml)
assert False, "Should raise an exception before getting here"
except TOMLError:
pass
def test_loading_complex_file_1():
toml = """
[main]
gid = 1
nid = 10
max_jobs = 100
message_id_file = "./.mid"
history_file = "./.history"
agent_controllers = ["http://localhost:8966/"]
[cmds]
[cmds.execute_js_py]
binary = "python2.7"
cwd = "./jumpscripts"
script = "{domain}/{name}.py"
[cmds.sync]
#syncthing extension
binary = "python2.7"
cwd = "./extensions/sync"
script = "{name}.py"
[cmds.sync.env]
PYTHONPATH = "../"
JUMPSCRIPTS_HOME = "../../jumpscripts"
SYNCTHING_URL = "http://localhost:8384"
[channel]
cmds = [0] # long polling from agent 0
[logging]
[logging.db]
type = "DB"
log_dir = "./logs"
levels = [2, 4, 7, 8, 9] # (all error messages) empty for all
[logging.ac]
type = "AC"
flush_int = 300 # seconds (5min)
batch_size = 1000 # max batch size, force flush if reached this count.
agent_controllers = [] # to all agents
levels = [2, 4, 7, 8, 9] # (all error messages) empty for all
[logging.console]
type = "console"
levels = [2, 4, 7, 8, 9]
[stats]
interval = 60 # seconds
agent_controllers = []
"""
contoml.loads(toml)
def test_parsing_multiline_strings_correctly():
toml = r'''multiline_empty_one = """"""
multiline_empty_two = """
"""
multiline_empty_three = """\
"""
multiline_empty_four = """\
\
\
"""
equivalent_one = "The quick brown fox jumps over the lazy dog."
equivalent_two = """
The quick brown \
fox jumps over \
the lazy dog."""
equivalent_three = """\
The quick brown \
fox jumps over \
the lazy dog.\
"""
'''
parsed = contoml.loads(toml)
assert parsed['']['multiline_empty_one'] == parsed['']['multiline_empty_two'] == \
parsed['']['multiline_empty_three'] == parsed['']['multiline_empty_four']
def test_table_with_pound_in_title():
toml = """["key#group"]
answer = 42"""
parsed = contoml.loads(toml)
assert parsed.primitive['key#group']['answer'] == 42
def test_accessing_deeply_nested_dicts():
t = """[cmds]
[cmds.sync]
#syncthing extension
binary = "python2.7"
cwd = "./extensions/sync"
script = "{name}.py"
[cmds.sync.env]
PYTHONPATH = "../"
JUMPSCRIPTS_HOME = "../../jumpscripts"
SYNCTHING_URL = "http://localhost:8384"
"""
f = contoml.loads(t)
assert f['cmds']['sync']['env']['SYNCTHING_URL'] == 'http://localhost:8384'
f['cmds']['sync']['env']['SYNCTHING_URL'] = 'Nowhere'
expected_toml = """[cmds]
[cmds.sync]
#syncthing extension
binary = "python2.7"
cwd = "./extensions/sync"
script = "{name}.py"
[cmds.sync.env]
PYTHONPATH = "../"
JUMPSCRIPTS_HOME = "../../jumpscripts"
SYNCTHING_URL = "Nowhere"
"""
assert expected_toml == f.dumps()
def test_weird_edge_case_1():
toml_text = """l = "t"
creativity = "on vacation"
"""
f = contoml.loads(toml_text)
assert f['']['l'] == 't'
def test_from_string(self, pipfile: str):
with open(os.path.join(self.data_dir, 'pipfiles', pipfile),
'r') as pipfile_file:
content = pipfile_file.read()
instance = Pipfile.from_string(content)
# Sometimes toml does not preserve inline tables causing to_string() fail. However, we produce valid toml.
assert instance.to_dict() == toml.loads(content).primitive
def test_parsing_section_with_indentation_and_comment_lines():
toml = """[main]
listen = ":8966"
redis_host = "localhost:6379"
redis_password = ""
[influxdb]
host = "localhost:8086"
db = "agentcontroller"
user = "******"
password = "******"
[handlers]
binary = "python2.7"
cwd = "./handlers"
[handlers.env]
PYTHONPATH = "/opt/jumpscale7/lib:../client"
SYNCTHING_URL = "http://localhost:8384/"
SYNCTHING_SHARED_FOLDER_ID = "jumpscripts"
#SYNCTHING_API_KEY = ""
REDIS_ADDRESS = "localhost"
REDIS_PORT = "6379"
#REDIS_PASSWORD = ""
"""
f = contoml.loads(toml)
assert f['handlers']['env']['REDIS_ADDRESS'] == 'localhost'
assert 'REDIS_PASSWORD' not in f['handlers']['env']
f['handlers']['env']['REDIS_PASSWORD'] = '******'
expected = """[main]
listen = ":8966"
redis_host = "localhost:6379"
redis_password = ""
[influxdb]
host = "localhost:8086"
db = "agentcontroller"
user = "******"
password = "******"
[handlers]
binary = "python2.7"
cwd = "./handlers"
[handlers.env]
PYTHONPATH = "/opt/jumpscale7/lib:../client"
SYNCTHING_URL = "http://localhost:8384/"
SYNCTHING_SHARED_FOLDER_ID = "jumpscripts"
#SYNCTHING_API_KEY = ""
REDIS_ADDRESS = "localhost"
REDIS_PORT = "6379"
REDIS_PASSWORD = "******"
#REDIS_PASSWORD = ""
"""
assert expected == f.dumps()
def test_one_entry_array_of_tables():
t = '''[[people]]
first_name = "Bruce"
last_name = "Springsteen"
'''
parsed = contoml.loads(t)
assert parsed['people'][0]['first_name'] == 'Bruce'
assert parsed['people'][0]['last_name'] == 'Springsteen'
def test_array_edge_cases():
# Parsing an empty array value
toml_text = """[section]
key = []"""
toml = contoml.loads(toml_text)
assert 'section' in toml
assert len(toml['section']['key']) == 0
def from_string(cls, pipfile_content: str):
"""Parse Pipfile from its string representation."""
_LOGGER.debug("Parsing Pipfile toml representation from string")
try:
parsed = toml.loads(pipfile_content)
parsed = parsed.primitive
except Exception as exc:
# We are transparent - Pipfile can be eigher TOML or JSON - try to parse any of these.
try:
parsed = json.loads(pipfile_content)
except Exception as exc:
raise PipfileParseError("Failed to parse provided Pipfile") from exc
return cls.from_dict(parsed)
def update_cargo_file_version(release_context, version):
with open(release_context.cargo_file, 'r+') as cargo_file:
cargo_content = contoml.loads(cargo_file.read())
cargo_content['package']['version'] = version
cargo_content.dump(release_context.cargo_file)
def read_cargo_file(release_context):
with open(release_context.cargo_file) as cargo_file:
cargo_content = contoml.loads(cargo_file.read())
return cargo_content['package']['version'], cargo_content['package']['name']
def update_cargo_file_version(release_context, version):
with open(release_context.cargo_file, 'r+') as cargo_file:
cargo_content = contoml.loads(cargo_file.read())
cargo_content['package']['version'] = version
cargo_content.dump(release_context.cargo_file)
def read_cargo_file(release_context):
with open(release_context.cargo_file) as cargo_file:
cargo_content = contoml.loads(cargo_file.read())
return (cargo_content['package']['version'], cargo_content['package']['name'])
def parsed_pipfile(self):
with open(self.pipfile_location) as f:
return contoml.loads(f.read())
def test_dump_nonascii_string():
content = 'name = "Stažené"\n'
toml_content = contoml.dumps(contoml.loads(content))
assert toml_content == content
def test_loading_toml_without_trailing_newline():
toml_text = '[main]\nname = "azmy"'
toml = contoml.loads(toml_text)
assert toml['main']['name'] == 'azmy'
def test_loading_an_empty_toml_source():
toml_text = ''
contoml.loads(toml_text)
def test_unicode_string_literals():
toml = u'answer = "δ"\n'
parsed = contoml.loads(toml)
assert parsed['']['answer'] == u"δ"
