def _testCScanner(self, test_name, data_filename, canonical_filename, file_input=False, Loader=yaml.Loader):
if file_input:
data = file(data_filename, 'r')
else:
data = file(data_filename, 'r').read()
tokens = list(yaml.scan(data, Loader=Loader))
ext_tokens = []
try:
if file_input:
data = file(data_filename, 'r')
for token in yaml.scan(data, Loader=yaml.CLoader):
ext_tokens.append(token)
self.failUnlessEqual(len(tokens), len(ext_tokens))
for token, ext_token in zip(tokens, ext_tokens):
self.failUnlessEqual(token.__class__, ext_token.__class__)
self.failUnlessEqual((token.start_mark.index, token.start_mark.line, token.start_mark.column),
(ext_token.start_mark.index, ext_token.start_mark.line, ext_token.start_mark.column))
self.failUnlessEqual((token.end_mark.index, token.end_mark.line, token.end_mark.column),
(ext_token.end_mark.index, ext_token.end_mark.line, ext_token.end_mark.column))
if hasattr(token, 'value'):
self.failUnlessEqual(token.value, ext_token.value)
except:
print
print "DATA:"
print file(data_filename, 'rb').read()
print "TOKENS:", tokens
print "EXT_TOKENS:", ext_tokens
raise
def _compare_scanners(py_data, c_data, verbose):
py_tokens = list(yaml.scan(py_data, Loader=yaml.PyLoader))
c_tokens = []
try:
for token in yaml.scan(c_data, Loader=yaml.CLoader):
c_tokens.append(token)
assert len(py_tokens) == len(c_tokens), (len(py_tokens), len(c_tokens))
for py_token, c_token in zip(py_tokens, c_tokens):
assert py_token.__class__ == c_token.__class__, (py_token, c_token)
if hasattr(py_token, 'value'):
assert py_token.value == c_token.value, (py_token, c_token)
if isinstance(py_token, yaml.StreamEndToken):
continue
py_start = (py_token.start_mark.index, py_token.start_mark.line,
py_token.start_mark.column)
py_end = (py_token.end_mark.index, py_token.end_mark.line,
py_token.end_mark.column)
c_start = (c_token.start_mark.index, c_token.start_mark.line,
c_token.start_mark.column)
c_end = (c_token.end_mark.index, c_token.end_mark.line,
c_token.end_mark.column)
assert py_start == c_start, (py_start, c_start)
assert py_end == c_end, (py_end, c_end)
finally:
if verbose:
print "PY_TOKENS:"
pprint.pprint(py_tokens)
print "C_TOKENS:"
pprint.pprint(c_tokens)
def _compare_scanners(py_data, c_data, verbose):
py_tokens = list(yaml.scan(py_data, Loader=yaml.PyLoader))
c_tokens = []
try:
for token in yaml.scan(c_data, Loader=yaml.CLoader):
c_tokens.append(token)
assert len(py_tokens) == len(c_tokens), (len(py_tokens), len(c_tokens))
for py_token, c_token in zip(py_tokens, c_tokens):
assert py_token.__class__ == c_token.__class__, (py_token, c_token)
if hasattr(py_token, 'value'):
assert py_token.value == c_token.value, (py_token, c_token)
if isinstance(py_token, yaml.StreamEndToken):
continue
py_start = (py_token.start_mark.index, py_token.start_mark.line, py_token.start_mark.column)
py_end = (py_token.end_mark.index, py_token.end_mark.line, py_token.end_mark.column)
c_start = (c_token.start_mark.index, c_token.start_mark.line, c_token.start_mark.column)
c_end = (c_token.end_mark.index, c_token.end_mark.line, c_token.end_mark.column)
assert py_start == c_start, (py_start, c_start)
assert py_end == c_end, (py_end, c_end)
finally:
if verbose:
print "PY_TOKENS:"
pprint.pprint(py_tokens)
print "C_TOKENS:"
pprint.pprint(c_tokens)
def load_jekyll_doc(file):
''' Load jekyll front matter and remaining content from a file.
Sample output:
{"title": "Greetings"}, "Hello world."
'''
# Check for presence of document separator.
file.seek(0)
if file.read(len(_marker)) != _marker:
raise Exception('No front-matter in %s' % getattr(file, 'name', None))
# Seek to just after the initial document separator.
file.seek(len(_marker))
for token in yaml.scan(file):
# Look for a token that shows we're about to reach the content.
if type(token) is yaml.DocumentStartToken:
# Seek to the beginning, and load the complete content.
file.seek(0)
chars = file.read().decode('utf8')
# Load the front matter as a document.
front_matter = yaml.safe_load(chars[:token.end_mark.index])
# Seek just after the document separator, and get remaining string.
content = chars[token.end_mark.index + len("\n" + _marker):]
return front_matter, content
raise Exception('Never found a yaml.DocumentStartToken')
def load_jekyll_doc(file):
''' Load jekyll front matter and remaining content from a file.
Sample output:
{"title": "Greetings"}, "Hello world."
'''
# Check for presence of document separator.
file.seek(0)
if file.read(len(_marker)) != _marker:
raise Exception('No front-matter in %s' % getattr(file, 'name', None))
# Seek to just after the initial document separator.
file.seek(len(_marker))
for token in yaml.scan(file):
# Look for a token that shows we're about to reach the content.
if type(token) is yaml.DocumentStartToken:
# Seek to the beginning, and load the complete content.
file.seek(0)
chars = file.read().decode('utf8')
# Load the front matter as a document.
front_matter = yaml.safe_load(chars[:token.end_mark.index])
# Seek just after the document separator, and get remaining string.
content = chars[token.end_mark.index + len("\n" + _marker):]
return front_matter, content
raise Exception('Never found a yaml.DocumentStartToken')
def get_imports(blueprint_file: typing.TextIO) -> dict:
level = 0
imports_token = None
import_lines = {}
blueprint_file.seek(0, 0)
for t in yaml.scan(blueprint_file):
if isinstance(t, (yaml.tokens.BlockMappingStartToken,
yaml.tokens.BlockSequenceStartToken,
yaml.tokens.FlowMappingStartToken,
yaml.tokens.FlowSequenceStartToken)):
level += 1
if isinstance(
t, (yaml.tokens.BlockEndToken, yaml.tokens.FlowMappingEndToken,
yaml.tokens.FlowSequenceEndToken)):
level -= 1
if isinstance(t, yaml.tokens.ScalarToken):
if level == 1 and t.value == 'imports':
imports_token = t
continue
token_length = t.end_mark.index - t.start_mark.index
if level >= 1 and imports_token and \
token_length < MAX_IMPORT_TOKEN_LENGTH:
import_lines[t.value] = {
START_POS: t.start_mark.index,
END_POS: t.end_mark.index,
}
if isinstance(t, yaml.tokens.KeyToken) and imports_token:
break
return import_lines
def highlight(self):
input = str(self.input.read())
substitutions = self.style.substitutions
tokens = yaml.scan(input)
events = yaml.parse(input)
markers = []
number = 0
for token in tokens:
number += 1
if token.start_mark.index != token.end_mark.index:
cls = token.__class__
if substitutions is not None:
if (cls, -1) in substitutions:
markers.append([
token.start_mark.index, +2, number,
substitutions[cls, -1]
])
if (cls, +1) in substitutions:
markers.append([
token.end_mark.index, -2, number,
substitutions[cls, +1]
])
number = 0
for event in events:
number += 1
cls = event.__class__
if substitutions is not None:
if (cls, -1) in substitutions:
markers.append([
event.start_mark.index, +1, number, substitutions[cls,
-1]
])
if (cls, +1) in substitutions:
markers.append([
event.end_mark.index, -1, number, substitutions[cls,
+1]
])
markers.sort()
markers.reverse()
chunks = []
position = len(input)
for index, weight1, weight2, substitution in markers:
if index < position:
chunk = input[index:position]
for substring, replacement in self.style.replaces:
chunk = chunk.replace(substring, replacement)
chunks.append(chunk)
position = index
chunks.append(substitution)
chunks.reverse()
result = u''.join(chunks)
if self.style.header is not None:
self.output.write(self.style.header)
try:
self.output.write(result.encode('utf-8'))
except Exception:
self.output.write(str(result))
#self.output.write(result)
if self.style.footer is not None:
self.output.write(self.style.footer)
def load_config(name, config_type=Configuration):
"""Load configuration data from a YAML file.
Valid configuration files are YAML files containing no custom types, no
sequences (lists), and with all mapping (dict) keys being valid python
identifiers.
Parameters
----------
name : str
Name of the configuration to load, which can either be a pre-defined
name or else the name of a yaml file (with extension .yaml) to load.
Pre-defined names are mapped to corresponding files in this package's
data/config/ directory.
Returns
-------
Configuration
Initialized configuration object.
Raises
------
ValueError
File name has wrong extension or does not exist.
RuntimeError
Configuration data failed a validation test.
"""
base_name, extension = os.path.splitext(name)
if extension not in ('', '.yaml'):
raise ValueError('Config file must have .yaml extension.')
if extension:
file_name = name
else:
file_name = astropy.utils.data._find_pkg_data_path(
'data/config/{0}.yaml'.format(name))
if not os.path.isfile(file_name):
raise ValueError('No such config file "{0}".'.format(file_name))
# Validate that all mapping keys are valid python identifiers.
valid_key = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*\Z')
with open(file_name) as f:
next_value_is_key = False
for token in yaml.scan(f):
if isinstance(
token,
(yaml.BlockSequenceStartToken, yaml.FlowSequenceStartToken)):
raise RuntimeError('Config sequences not implemented yet.')
if next_value_is_key:
if not isinstance(token, yaml.ScalarToken):
raise RuntimeError(
'Invalid config key type: {0}'.format(token))
if not valid_key.match(token.value):
raise RuntimeError(
'Invalid config key name: {0}'.format(token.value))
next_value_is_key = isinstance(token, yaml.KeyToken)
with open(file_name) as f:
config = config_type(yaml.safe_load(f))
return config
def load_config(name, config_type=Configuration):
"""Load configuration data from a YAML file.
Valid configuration files are YAML files containing no custom types, no
sequences (lists), and with all mapping (dict) keys being valid python
identifiers.
Parameters
----------
name : str
Name of the configuration to load, which can either be a pre-defined
name or else the name of a yaml file (with extension .yaml) to load.
Pre-defined names are mapped to corresponding files in this package's
data/config/ directory.
Returns
-------
Configuration
Initialized configuration object.
Raises
------
ValueError
File name has wrong extension or does not exist.
RuntimeError
Configuration data failed a validation test.
"""
base_name, extension = os.path.splitext(name)
if extension not in ('', '.yaml'):
raise ValueError('Config file must have .yaml extension.')
if extension:
file_name = name
else:
file_name = astropy.utils.data._find_pkg_data_path(
'data/config/{0}.yaml'.format(name))
if not os.path.isfile(file_name):
raise ValueError('No such config file "{0}".'.format(file_name))
# Validate that all mapping keys are valid python identifiers.
valid_key = re.compile('^[a-zA-Z_][a-zA-Z0-9_]*\Z')
with open(file_name) as f:
next_value_is_key = False
for token in yaml.scan(f):
if isinstance(
token,
(yaml.BlockSequenceStartToken, yaml.FlowSequenceStartToken)):
raise RuntimeError('Config sequences not implemented yet.')
if next_value_is_key:
if not isinstance(token, yaml.ScalarToken):
raise RuntimeError(
'Invalid config key type: {0}'.format(token))
if not valid_key.match(token.value):
raise RuntimeError(
'Invalid config key name: {0}'.format(token.value))
next_value_is_key = isinstance(token, yaml.KeyToken)
with open(file_name) as f:
return config_type(yaml.safe_load(f))
def highlight(self):
input = self.input.read()
if input.startswith(codecs.BOM_UTF16_LE):
input = six.text_type(input, 'utf-16-le')
elif input.startswith(codecs.BOM_UTF16_BE):
input = six.text_type(input, 'utf-16-be')
else:
input = six.text_type(input, 'utf-8')
substitutions = self.style.substitutions
tokens = yaml.scan(input)
events = yaml.parse(input)
markers = []
number = 0
for token in tokens:
number += 1
if token.start_mark.index != token.end_mark.index:
cls = token.__class__
if (cls, -1) in substitutions:
markers.append([
token.start_mark.index, +2, number, substitutions[cls,
-1]
])
if (cls, +1) in substitutions:
markers.append([
token.end_mark.index, -2, number, substitutions[cls,
+1]
])
number = 0
for event in events:
number += 1
cls = event.__class__
if (cls, -1) in substitutions:
markers.append([
event.start_mark.index, +1, number, substitutions[cls, -1]
])
if (cls, +1) in substitutions:
markers.append(
[event.end_mark.index, -1, number, substitutions[cls, +1]])
markers.sort()
markers.reverse()
chunks = []
position = len(input)
for index, weight1, weight2, substitution in markers:
if index < position:
chunk = input[index:position]
for substring, replacement in self.style.replaces:
chunk = chunk.replace(substring, replacement)
chunks.append(chunk)
position = index
chunks.append(substitution)
chunks.reverse()
result = ''.join(chunks)
if self.style.header:
self.output.write(self.style.header)
self.output.write(result.encode('utf-8'))
if self.style.footer:
self.output.write(self.style.footer)
def test_scanner(data_filename, canonical_filename, verbose=False):
for filename in [data_filename, canonical_filename]:
tokens = []
try:
for token in yaml.scan(open(filename, 'rb')):
tokens.append(token.__class__.__name__)
finally:
if verbose:
pprint.pprint(tokens)
def test_scanner(data_filename, canonical_filename, verbose=False):
for filename in [data_filename, canonical_filename]:
tokens = []
try:
for token in yaml.scan(open(filename, 'rb')):
tokens.append(token.__class__.__name__)
finally:
if verbose:
pprint.pprint(tokens)
def tokenize(text, loader=yaml.SafeLoader):
last_token = yaml.ValueToken(None, None)
for token in yaml.scan(text, loader):
start = token.start_mark.index
end = token.end_mark.index
if isinstance(token, yaml.TagToken):
yield start, end, token
elif isinstance(token, yaml.ScalarToken):
yield start, end, last_token
elif isinstance(token, (yaml.KeyToken, yaml.ValueToken)):
last_token = token
def highlight(self):
input = self.input.read()
if input.startswith(codecs.BOM_UTF16_LE):
input = unicode(input, 'utf-16-le')
elif input.startswith(codecs.BOM_UTF16_BE):
input = unicode(input, 'utf-16-be')
else:
input = unicode(input, 'utf-8')
substitutions = self.style.substitutions
tokens = yaml.scan(input)
events = yaml.parse(input)
markers = []
number = 0
for token in tokens:
number += 1
if token.start_mark.index != token.end_mark.index:
cls = token.__class__
if (cls, -1) in substitutions:
markers.append([token.start_mark.index, +2, number,
substitutions[cls, -1]])
if (cls, +1) in substitutions:
markers.append([token.end_mark.index, -2, number,
substitutions[cls, +1]])
number = 0
for event in events:
number += 1
cls = event.__class__
if (cls, -1) in substitutions:
markers.append([event.start_mark.index, +1, number,
substitutions[cls, -1]])
if (cls, +1) in substitutions:
markers.append([event.end_mark.index, -1, number,
substitutions[cls, +1]])
markers.sort()
markers.reverse()
chunks = []
position = len(input)
for (index, weight1, weight2, substitution) in markers:
if index < position:
chunk = input[index:position]
for (substring, replacement) in self.style.replaces:
chunk = chunk.replace(substring, replacement)
chunks.append(chunk)
position = index
chunks.append(substitution)
chunks.reverse()
result = u''.join(chunks)
if self.style.header:
self.output.write(self.style.header)
self.output.write(result.encode('utf-8'))
if self.style.footer:
self.output.write(self.style.footer)
def _testCScanner(self,
test_name,
data_filename,
canonical_filename,
file_input=False,
Loader=yaml.Loader):
if file_input:
data = file(data_filename, 'r')
else:
data = file(data_filename, 'r').read()
tokens = list(yaml.scan(data, Loader=Loader))
ext_tokens = []
try:
if file_input:
data = file(data_filename, 'r')
for token in yaml.scan(data, Loader=yaml.CLoader):
ext_tokens.append(token)
self.failUnlessEqual(len(tokens), len(ext_tokens))
for token, ext_token in zip(tokens, ext_tokens):
self.failUnlessEqual(token.__class__, ext_token.__class__)
self.failUnlessEqual(
(token.start_mark.index, token.start_mark.line,
token.start_mark.column),
(ext_token.start_mark.index, ext_token.start_mark.line,
ext_token.start_mark.column))
self.failUnlessEqual(
(token.end_mark.index, token.end_mark.line,
token.end_mark.column),
(ext_token.end_mark.index, ext_token.end_mark.line,
ext_token.end_mark.column))
if hasattr(token, 'value'):
self.failUnlessEqual(token.value, ext_token.value)
except:
print
print "DATA:"
print file(data_filename, 'rb').read()
print "TOKENS:", tokens
print "EXT_TOKENS:", ext_tokens
raise
def test_tokens(data_filename, tokens_filename, verbose=False):
tokens1 = []
tokens2 = open(tokens_filename, 'r').read().split()
try:
for token in yaml.scan(open(data_filename, 'rb')):
if not isinstance(token, (yaml.StreamStartToken, yaml.StreamEndToken)):
tokens1.append(_replaces[token.__class__])
finally:
if verbose:
print("TOKENS1:", ' '.join(tokens1))
print("TOKENS2:", ' '.join(tokens2))
assert len(tokens1) == len(tokens2), (tokens1, tokens2)
for token1, token2 in zip(tokens1, tokens2):
assert token1 == token2, (token1, token2)
def test_tokens(data_filename, tokens_filename, verbose=False):
tokens1 = []
tokens2 = open(tokens_filename, 'rb').read().split()
try:
for token in yaml.scan(open(data_filename, 'rb')):
if not isinstance(token,
(yaml.StreamStartToken, yaml.StreamEndToken)):
tokens1.append(_replaces[token.__class__])
finally:
if verbose:
print "TOKENS1:", ' '.join(tokens1)
print "TOKENS2:", ' '.join(tokens2)
assert len(tokens1) == len(tokens2), (tokens1, tokens2)
for token1, token2 in zip(tokens1, tokens2):
assert token1 == token2, (token1, token2)
def remove_header(text):
"""Remove a header from the text, returning the remaining raw text."""
body_start = 0
n_document_start_tokens = 0
for elem in yaml.scan(text):
if isinstance(elem, yaml.DocumentStartToken):
n_document_start_tokens += 1
if n_document_start_tokens == 2:
if len(text) > elem.end_mark.pointer + 1:
body_start = elem.end_mark.pointer + 1
break
return text[body_start:]
def load_yaml_and_body(file):
for token in yaml.scan(file):
# Look for a token that shows we're about to reach the content.
if type(token) is yaml.DocumentStartToken:
# Seek to the beginning, and load the complete content.
file.seek(0)
chars = file.read().decode('utf-8')
# Load the front matter as a document.
front_matter = yaml.safe_load(chars[:token.end_mark.index])
# Seek just after the document separator, and get remaining string.
content = chars[token.end_mark.index + len("\n" + _marker):]
return front_matter, content
raise Exception('Couldn\'t find yaml.DocumentStartToken when loading a file.')
def load_yaml_and_body(file):
for token in yaml.scan(file):
# Look for a token that shows we're about to reach the content.
if type(token) is yaml.DocumentStartToken:
# Seek to the beginning, and load the complete content.
file.seek(0)
chars = file.read().decode('utf-8')
# Load the front matter as a document.
front_matter = yaml.safe_load(chars[:token.end_mark.index])
# Seek just after the document separator, and get remaining string.
content = chars[token.end_mark.index + len("\n" + _marker):]
return front_matter, content
raise Exception(
'Couldn\'t find yaml.DocumentStartToken when loading a file.')
def load(
document,
allow_flow_style=False,
allow_tag_tokens=False,
allow_anchor_tokens=False
):
"""Return a YAMLItem object representation of a dumb yaml document"""
for token in yaml.scan(document):
if type(token) == yaml.tokens.FlowMappingStartToken and \
not bool(allow_flow_style):
raise exceptions.FlowMappingDisallowed(token)
if type(token) == yaml.tokens.TagToken and \
not bool(allow_tag_tokens):
raise exceptions.TagTokenDisallowed(token)
if type(token) == yaml.tokens.AnchorToken and \
not bool(allow_anchor_tokens):
raise exceptions.AnchorTokenDisallowed(token)
return YAMLNode(yaml.load(document, Loader=DumbLoader))
def _initialize(self, file_name=None):
"""Initialize a configuration data structure from a YAML file.
"""
if file_name is None:
file_name = 'config.yaml'
# Remember the file name since it is not allowed to change.
self.file_name = file_name
# Locate the config file in our pkg data/ directory if no path is given.
if os.path.split(file_name)[0] == '':
full_path = astropy.utils.data._find_pkg_data_path(
os.path.join('data', file_name))
else:
full_path = file_name
# Validate that all mapping keys are valid python identifiers
# and that there are no embedded sequences.
valid_key = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*\Z')
with open(full_path) as f:
next_value_is_key = False
for token in yaml.scan(f):
if isinstance(
token,
(yaml.BlockSequenceStartToken,
yaml.FlowSequenceStartToken)):
raise RuntimeError('Config sequences not supported.')
if next_value_is_key:
if not isinstance(token, yaml.ScalarToken):
raise RuntimeError(
'Invalid config key type: {0}'.format(token))
if not valid_key.match(token.value):
raise RuntimeError(
'Invalid config key name: {0}'.format(token.value))
next_value_is_key = isinstance(token, yaml.KeyToken)
# Load the config data into memory.
with open(full_path) as f:
Node.__init__(self, yaml.safe_load(f))
# Output path is not set until it is first used.
self._output_path = None
def test_get_line_indent(self):
tokens = list(yaml.scan('a: 1\n'
'b:\n'
' - c: [2, 3, {d: 4}]\n'))
self.assertEqual(tokens[3].value, 'a')
self.assertEqual(tokens[5].value, '1')
self.assertEqual(tokens[7].value, 'b')
self.assertEqual(tokens[13].value, 'c')
self.assertEqual(tokens[16].value, '2')
self.assertEqual(tokens[18].value, '3')
self.assertEqual(tokens[22].value, 'd')
self.assertEqual(tokens[24].value, '4')
for i in (3, 5):
self.assertEqual(get_line_indent(tokens[i]), 0)
for i in (7,):
self.assertEqual(get_line_indent(tokens[i]), 0)
for i in (13, 16, 18, 22, 24):
self.assertEqual(get_line_indent(tokens[i]), 2)
def build_tree(data):
if isinstance(data, str):
data = json.loads(data)
yaml_text = yaml.safe_dump(data,
allow_unicode=True,
default_flow_style=False)
if six.PY2:
yaml_text = yaml_text.decode("utf-8")
if not settings.colored_output:
return yaml_text
pairs = []
in_key = False
for token in yaml.scan(yaml_text):
if token.__class__ == yaml.tokens.KeyToken:
in_key = True
continue
if in_key and yaml.tokens.ScalarToken == token.__class__:
pairs.append((token.start_mark.index, token.end_mark.index))
in_key = False
if not pairs:
return yaml_text
last_pos = pairs[0][0]
result = yaml_text[:last_pos]
for start, end in pairs:
result += yaml_text[last_pos:start] + "\x1b[31m" + yaml_text[
start:end] + "\x1b[39m"
last_pos = end
result += yaml_text[last_pos:]
return result
def compile(script):
def _expect_token(token, token_type, msg="invalid syntax"):
if not isinstance(token, token_type):
raise ScriptParseError(script.name, token, msg)
return token
try:
tokens = yaml.scan(script)
_expect_token(tokens.send(None), yaml.StreamStartToken)
_expect_token(tokens.send(None), yaml.BlockMappingStartToken)
commands = []
for token in tokens:
if isinstance(token, yaml.BlockEndToken):
break
_expect_token(token, yaml.KeyToken)
key_tok = _expect_token(tokens.send(None), yaml.ScalarToken)
command_name = key_tok.value
_expect_token(tokens.send(None), yaml.ValueToken)
value_tok = _expect_token(tokens.send(None), yaml.ScalarToken)
command_arg = value_tok.value
cmd_class = SCRIPT_COMMANDS.get(command_name)
if not cmd_class:
raise ScriptParseError(script.name, key_tok,
("Unknown command '%s'" % command_name))
try:
commands.append(cmd_class(command_arg))
except Exception as e:
raise ScriptParseError(script.name, value_tok, str(e))
_expect_token(tokens.send(None), yaml.StreamEndToken)
except yaml.scanner.ScannerError as e:
raise ScriptParseError(script.name, e, e.problem)
return Sequence(commands)
def keypaths(document, row, column):
text = document
row = int(row) - 1
column = int(column) - 1
#print(text)
paths = []
lastKey = None
mappingStart = False
for token in yaml.scan(text):
#print("line:%d,column:%d" % (token.start_mark.line,token.start_mark.column))
#print("token:%s" % repr(token))
#print("start_mark:%s \nend_mark:%s" % (token.start_mark, token.end_mark))
if token.start_mark.line > row or (token.start_mark.line == row and
token.start_mark.column > column):
return paths
if isinstance(token, yaml.tokens.KeyToken):
isKey = True
isValue = False
else:
if isinstance(token, yaml.tokens.ValueToken):
if len(paths) > 0:
paths[-1] = lastKey.value
if isinstance(token, yaml.tokens.ScalarToken) and isKey:
lastKey = token
if mappingStart is True:
paths.append(token.value)
mappingStart = False
if isinstance(token, yaml.tokens.BlockEndToken):
if len(paths) > 0 and not (token.start_mark.line == row and
token.start_mark.column > column):
paths.pop()
if isinstance(token, yaml.tokens.BlockMappingStartToken):
mappingStart = True
isKey = False
return paths
def build_tree(data):
if isinstance(data, str):
data = json.loads(data)
yaml_text = yaml.safe_dump(data, allow_unicode=True, default_flow_style=False)
if six.PY2:
yaml_text = yaml_text.decode("utf-8")
if not settings.colored_output:
return yaml_text
pairs = []
in_key = False
for token in yaml.scan(yaml_text):
if token.__class__ == yaml.tokens.KeyToken:
in_key = True
continue
if in_key and yaml.tokens.ScalarToken == token.__class__:
pairs.append((token.start_mark.index, token.end_mark.index))
in_key = False
if not pairs:
return yaml_text
last_pos = pairs[0][0]
result = yaml_text[:last_pos]
for start, end in pairs:
result += yaml_text[last_pos:start] + "\x1b[31m" + yaml_text[start:end] + "\x1b[39m"
last_pos = end
result += yaml_text[last_pos:]
return result
def add(self, buffer):
try:
config = yaml.load(buffer)
if not config: # empty file
return
except yaml.YAMLError as e:
s = None
if hasattr(e, 'problem_mark'):
mark = e.problem_mark
s = "error at line %s column %s" % (mark.line + 1,
mark.column + 1)
else:
s = str(e)
raise SystemConfigSyntaxException(s)
# scan the buffer to determine section order, which would be
# used to guarantee instrument initialization order
sectionsOrder = []
for token in yaml.scan(buffer):
# get all ScalarTokens
if isinstance(token, yaml.ScalarToken):
# add to order if is a known section and unique to order
if token.value in self._sections and token.value not in sectionsOrder:
sectionsOrder.append(token.value)
# parse chimera section first, to get host/port or set defaults
for type, values in list(config.items()):
if type.lower() == "chimera":
try:
if values["host"] == "0.0.0.0":
values["host"] = None
except KeyError:
pass
self.chimera.update(values)
# BIGGG FIXME: read all files before create Locations,
# to avoid this hack
if "host" in values or "port" in values:
# host/port changed
for l in self.instruments + self.controllers:
l._host = values["host"]
l._port = values["port"]
del config[type]
break
# ensure host/port setup on chimera
if "host" not in self.chimera:
self.chimera["host"] = MANAGER_DEFAULT_HOST
if "port" not in self.chimera:
self.chimera["port"] = MANAGER_DEFAULT_PORT
objects = {}
for type, values in list(config.items()):
key = type.lower()
objects[key] = []
if not isinstance(values, list):
values = [values]
for instance in values:
loc = self._parseLocation(key, instance)
objects[key].append(loc)
# create instruments list ordered by sectionsOrder list created above
for section in sectionsOrder:
if section in self._instrumentsSections and section in objects:
self.instruments += objects[section]
if section in self._controllersSections and section in objects:
self.controllers += objects[section]
# always use a single site from the last added file
if "site" in objects:
self.sites = [objects["site"][0]]
return True
def canonical_scan(stream):
return yaml.scan(stream, Loader=CanonicalLoader)
def canonical_scan(stream):
return yaml.scan(stream, Loader=CanonicalLoader)
class SystemConfig(object):
"""
Chimera configuration system
============================
Chimera uses YAML format to the configutation system. YAML use
basic syntax to allow sequences and maps to be easily parsed.
A map (a key=value sequence) is defined using:
key: value
otherkey: value
will be mapped to {'key': 'value', 'otherkey': 'value'}
A sequence is defined as (in this case a simple progression)
- 1
- 2
- 3
will be mapped to [1,2,3]
Chimera uses both maps and sequences. It use keys that maps to
Instrmuments/Controllers names. And for each key, the value can be
either another map or a sequence of maps.
The following example defines a instrument with a few parameters:
instrument:
parameter_1: value1
parameter_2: value2
parameter_3: value3
To define more than one instance, just pass an sequence as the
maps value, as in:
instrument:
- parameter_1: value1
parameter_2: value2
parameter_3: value3
- parameter_1: other1
parameter_2: other2
parameter_3: other3
You could also pass another map as a parameter value, as in:
instrument:
parameter_1:
parameterkey: paramatervalue
this would be mapped to:
{'InstrumentType': {'parameter1': {'parameterkey': 'parametervalue'}
Chimera accepts definition of instruments and controllers. Besides
this, there are specials shortcut to most common object types
(like telescopes).
For each object, Chimera accepts a set os parameters plus any
other parameter specific to the given object type. The deault
parameters are:
name: a_valid_chimera_name_for_the_instance
type: ChimeraObject type
host: where this object is/or will be located.
port: port where to find the object on the given host
For special shortcuts the type would be guessed, so the folling
two entries are equivalent:
instrument:
type: Telescope
name: meade
device: /dev/ttyS0
telescope:
name: meade
device: /dev/ttyS0
"""
@staticmethod
def fromString(string):
s = SystemConfig()
s.add(string)
return s
@staticmethod
def fromFile(filename):
s = SystemConfig()
s.add(open(filename, "r").read())
return s
@staticmethod
def fromDefault():
return SystemConfig.fromFile(SYSTEM_CONFIG_DEFAULT_FILENAME)
def __init__(self):
# primitives
self.chimera = {}
self.sites = []
self.instruments = []
self.controllers = []
# specials
self._specials = [
"telescope", "telescopes", "camera", "cameras", "filterwheel",
"filterwheels", "dome", "domes", "focuser", "focusers"
]
self._instrumentsSections = self._specials + [
"instrument", "instruments"
]
self._controllersSections = ["controller", "controllers"]
self._sections = self._instrumentsSections + self._controllersSections + [
"site", "chimera"
]
# to create nice numbered names for objects without a name
self._useCount = {}
self.chimera["host"] = MANAGER_DEFAULT_HOST
self.chimera["port"] = MANAGER_DEFAULT_PORT
def add(self, buffer):
try:
config = yaml.load(buffer)
if not config: # empty file
return
except yaml.YAMLError, e:
s = None
if hasattr(e, 'problem_mark'):
mark = e.problem_mark
s = "error at line %s column %s" % (mark.line + 1,
mark.column + 1)
else:
s = str(e)
raise SystemConfigSyntaxException(s)
# scan the buffer to determine section order, which would be
# used to guarantee instrument initialization order
sectionsOrder = []
for token in yaml.scan(buffer):
# get all ScalarTokens
if isinstance(token, yaml.ScalarToken):
# add to order if is a known section and unique to order
if token.value in self._sections and token.value not in sectionsOrder:
sectionsOrder.append(token.value)
# parse chimera section first, to get host/port or set defaults
for type, values in config.items():
if type.lower() == "chimera":
self.chimera.update(values)
# BIGGG FIXME: read all files before create Locations,
# to avoid this hack
if "host" in values or "port" in values:
# host/port changed
for l in self.instruments + self.controllers:
l._host = values["host"]
l._port = values["port"]
del config[type]
break
# ensure host/port setup on chimera
if "host" not in self.chimera:
self.chimera["host"] = MANAGER_DEFAULT_HOST
if "port" not in self.chimera:
self.chimera["port"] = MANAGER_DEFAULT_PORT
objects = {}
for type, values in config.items():
key = type.lower()
objects[key] = []
if not isinstance(values, list):
values = [values]
for instance in values:
loc = self._parseLocation(key, instance)
objects[key].append(loc)
# create instruments list ordered by sectionsOrder list created above
for section in sectionsOrder:
if section in self._instrumentsSections and section in objects:
self.instruments += objects[section]
if section in self._controllersSections and section in objects:
self.controllers += objects[section]
# always use a single site from the last added file
if "site" in objects:
self.sites = [objects["site"][0]]
return True
#!/usr/bin/python
import yaml
with open('items.yaml') as f:
data = yaml.scan(f, Loader=yaml.FullLoader)
for token in data:
print(token)
def load_config(name, config_type=Configuration):
"""Load configuration data from a YAML file.
Valid configuration files are YAML files containing no custom types, no
sequences (lists), and with all mapping (dict) keys being valid python
identifiers.
Parameters
----------
name : str
Name of the configuration to load, which can either be a pre-defined
name or else the name of a yaml file (with extension .yaml) to load.
Pre-defined names are mapped to corresponding files in this package's
data/config/ directory.
Returns
-------
Configuration
Initialized configuration object.
Raises
------
ValueError
File name has wrong extension or does not exist.
RuntimeError
Configuration data failed a validation test.
"""
base_name, extension = os.path.splitext(name)
if extension not in ('', '.yaml'):
raise ValueError('Config file must have .yaml extension.')
if extension:
file_name = name
else:
file_name = astropy.utils.data._find_pkg_data_path(
'data/config/{0}.yaml'.format(name))
if not os.path.isfile(file_name):
raise ValueError('No such config file "{0}".'.format(file_name))
# Validate that all mapping keys are valid python identifiers.
valid_key = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*\Z')
with open(file_name) as f:
next_value_is_key = False
for token in yaml.scan(f):
# if isinstance(
# token,
# (yaml.BlockSequenceStartToken, yaml.FlowSequenceStartToken)):
# raise RuntimeError('Config sequences not implemented yet.')
if next_value_is_key:
if not isinstance(token, yaml.ScalarToken):
raise RuntimeError(
'Invalid config key type: {0}'.format(token))
if not valid_key.match(token.value):
raise RuntimeError('Invalid config key name: {0}'.format(
token.value))
next_value_is_key = isinstance(token, yaml.KeyToken)
with open(file_name) as f:
conf = config_type(yaml.safe_load(f))
# Enforces type of data member
# TODO: Add warning/error checking for missing config or IO members
# This includes the simulations and IO fields. They must exist even if unspecified.
# - Perhaps read from default if they are missing?
if isinstance(conf.simulation.interactions, str):
temp = conf.simulation.interactions.replace(' ', '').split(',')
if len(temp) > 1:
conf.simulation.interactions = temp
else:
conf.simulation.interactions = temp[0]
if isinstance(conf.simulation.flavors, str):
temp = conf.simulation.flavors.replace(' ', '').split(',')
if len(temp) > 1:
conf.simulation.flavors = temp
else:
conf.simulation.flavors = temp[0]
return conf
def get_show_dir_branch(self, branch_tag, create_branch=False, resolve_branch_hash=None):
'''
Parse the show_dir_config block from the show yaml, and build a list of tagged branches.
@param branch_tag specify the tag name of the particular branch, this tag is in the show_yaml.
@param create_branch create the branch if doesn't already exist
@param resovle_branch_hash any branch tokens that needs to be resolved.
@return a list representing the hierarchy path of the branch.
ex: get_show_dir_branch( branch_tag="branch_asset_root" )
> '?project:?prod_dir_type:assets'
'''
if resolve_branch_hash is None:
resolve_branch_hash = {}
if self._show_dir_branch==None:
f = file(self._config_yaml_path, 'r')
# use to walk the show dir stack. If this is None, it means it has yet encounter the show_dir_config block
self._show_dir_branch = {}
dir_walk_stack = None
for t in yaml.scan(f):
if type(t) == yaml.tokens.ScalarToken and t.value=='show_dir_config':
dir_walk_stack = []
if dir_walk_stack==None:
continue
if type(t) == yaml.tokens.BlockMappingStartToken:
dir_walk_stack.append(None)
if type(t) == yaml.tokens.ScalarToken and len(dir_walk_stack):
dir_walk_stack[-1] = t.value
if type(t) == yaml.tokens.BlockEndToken:
dir_walk_stack.pop(-1)
if len(dir_walk_stack)==0: # complete parsing
break
if type(t) == yaml.tokens.AnchorToken:
self._show_dir_branch[t.value] = copy.copy(dir_walk_stack)
if branch_tag in self._show_dir_branch:
# some defaults
if 'project' not in resolve_branch_hash: resolve_branch_hash['project'] = current_show()
#if 'prod_dir_type' not in resolve_branch_hash: resolve_branch_hash['prod_dir_type'] = 'publish'
if 'artist' not in resolve_branch_hash: resolve_branch_hash['artist'] = 'common'
# if publish then don't need artist in path
branch_string = ':'.join( self._show_dir_branch[branch_tag] )
if resolve_branch_hash.get('prod_dir_type') == 'publish' and '?artist' in branch_string:
branch_string = branch_string.replace(':?artist:', ':')
# create the branch if necessary
if create_branch:
from tbe_core.utils import shot_dir_setup
return get_show_dir_root() + '/' + shot_dir_setup.create_branch(
get_show_dir_root(),
branch_string,
resolve_branch_hash
)
else:
return branch_string
else:
log.warning("Warning: Failed to find the branch tag '%s' from the show directory config in '%s'. Available: \n%s" % (
branch_tag, self._config_yaml_path, self._show_dir_branch.keys() ))
return None
def prepare_transform(self):
self.tokens = yaml.scan(self.output())
