def setUp(self):
stream = NameStringIO(u"""
- hosts: localhost
vars:
number: 1
string: Assible
utf8_string: Cafè Eñyei
dictionary:
webster: daniel
oed: oxford
list:
- a
- b
- 1
- 2
tasks:
- name: Test case
ping:
data: "{{ utf8_string }}"
- name: Test 2
ping:
data: "Cafè Eñyei"
- name: Test 3
command: "printf 'Cafè Eñyei\\n'"
""")
self.play_filename = '/path/to/myplay.yml'
stream.name = self.play_filename
self.loader = AssibleLoader(stream)
self.data = self.loader.get_single_data()
def test_parse_number(self):
stream = StringIO(u"""
1
""")
loader = AssibleLoader(stream, 'myfile.yml')
data = loader.get_single_data()
self.assertEqual(data, 1)
def test_parse_utf8_string(self):
stream = StringIO(u"""
Cafè Eñyei
""")
loader = AssibleLoader(stream, 'myfile.yml')
data = loader.get_single_data()
self.assertEqual(data, u'Cafè Eñyei')
self.assertIsInstance(data, text_type)
self.assertEqual(data.assible_pos, ('myfile.yml', 2, 17))
def _safe_load(stream, file_name=None, vault_secrets=None):
''' Implements yaml.safe_load(), except using our custom loader class. '''
loader = AssibleLoader(stream, file_name, vault_secrets)
try:
return loader.get_single_data()
finally:
try:
loader.dispose()
except AttributeError:
pass # older versions of yaml don't have dispose function, ignore
def test_parse_list(self):
stream = StringIO(u"""
- a
- b
""")
loader = AssibleLoader(stream, 'myfile.yml')
data = loader.get_single_data()
self.assertEqual(data, [u'a', u'b'])
self.assertEqual(len(data), 2)
self.assertIsInstance(data[0], text_type)
self.assertEqual(data.assible_pos, ('myfile.yml', 2, 17))
self.assertEqual(data[0].assible_pos, ('myfile.yml', 2, 19))
self.assertEqual(data[1].assible_pos, ('myfile.yml', 3, 19))
def test_parse_dict(self):
stream = StringIO(u"""
webster: daniel
oed: oxford
""")
loader = AssibleLoader(stream, 'myfile.yml')
data = loader.get_single_data()
self.assertEqual(data, {'webster': 'daniel', 'oed': 'oxford'})
self.assertEqual(len(data), 2)
self.assertIsInstance(list(data.keys())[0], text_type)
self.assertIsInstance(list(data.values())[0], text_type)
# Beginning of the first key
self.assertEqual(data.assible_pos, ('myfile.yml', 2, 17))
self.assertEqual(data[u'webster'].assible_pos, ('myfile.yml', 2, 26))
self.assertEqual(data[u'oed'].assible_pos, ('myfile.yml', 3, 22))
def read_docstring(filename, verbose=True, ignore_errors=True):
"""
Search for assignment of the DOCUMENTATION and EXAMPLES variables in the given file.
Parse DOCUMENTATION from YAML and return the YAML doc or None together with EXAMPLES, as plain text.
"""
data = {
'doc': None,
'plainexamples': None,
'returndocs': None,
'metadata': None, # NOTE: not used anymore, kept for compat
'seealso': None,
}
string_to_vars = {
'DOCUMENTATION': 'doc',
'EXAMPLES': 'plainexamples',
'RETURN': 'returndocs',
'ASSIBLE_METADATA': 'metadata', # NOTE: now unused, but kept for backwards compat
}
try:
with open(filename, 'rb') as b_module_data:
M = ast.parse(b_module_data.read())
for child in M.body:
if isinstance(child, ast.Assign):
for t in child.targets:
try:
theid = t.id
except AttributeError:
# skip errors can happen when trying to use the normal code
display.warning("Failed to assign id for %s on %s, skipping" % (t, filename))
continue
if theid in string_to_vars:
varkey = string_to_vars[theid]
if isinstance(child.value, ast.Dict):
data[varkey] = ast.literal_eval(child.value)
else:
if theid == 'EXAMPLES':
# examples 'can' be yaml, but even if so, we dont want to parse as such here
# as it can create undesired 'objects' that don't display well as docs.
data[varkey] = to_text(child.value.s)
else:
# string should be yaml if already not a dict
data[varkey] = AssibleLoader(child.value.s, file_name=filename).get_single_data()
display.debug('assigned: %s' % varkey)
except Exception:
if verbose:
display.error("unable to parse %s" % filename)
if not ignore_errors:
raise
return data
def test_parse_short_dict(self):
stream = StringIO(u"""{"foo": "bar"}""")
loader = AssibleLoader(stream, 'myfile.yml')
data = loader.get_single_data()
self.assertEqual(data, dict(foo=u'bar'))
self.assertEqual(data.assible_pos, ('myfile.yml', 1, 1))
self.assertEqual(data[u'foo'].assible_pos, ('myfile.yml', 1, 9))
stream = StringIO(u"""foo: bar""")
loader = AssibleLoader(stream, 'myfile.yml')
data = loader.get_single_data()
self.assertEqual(data, dict(foo=u'bar'))
self.assertEqual(data.assible_pos, ('myfile.yml', 1, 1))
self.assertEqual(data[u'foo'].assible_pos, ('myfile.yml', 1, 6))
def read_docstub(filename):
"""
Quickly find short_description using string methods instead of node parsing.
This does not return a full set of documentation strings and is intended for
operations like assible-doc -l.
"""
in_documentation = False
capturing = False
indent_detection = ''
doc_stub = []
with open(filename, 'r') as t_module_data:
for line in t_module_data:
if in_documentation:
# start capturing the stub until indentation returns
if capturing and line.startswith(indent_detection):
doc_stub.append(line)
elif capturing and not line.startswith(indent_detection):
break
elif line.lstrip().startswith('short_description:'):
capturing = True
# Detect that the short_description continues on the next line if it's indented more
# than short_description itself.
indent_detection = ' ' * (len(line) - len(line.lstrip()) + 1)
doc_stub.append(line)
elif line.startswith('DOCUMENTATION') and '=' in line:
in_documentation = True
short_description = r''.join(doc_stub).strip().rstrip('.')
data = AssibleLoader(short_description, file_name=filename).get_single_data()
return data
def test_front_matter(self):
stream = StringIO(u"""---\nfoo: bar""")
loader = AssibleLoader(stream, 'myfile.yml')
data = loader.get_single_data()
self.assertEqual(data, dict(foo=u'bar'))
self.assertEqual(data.assible_pos, ('myfile.yml', 2, 1))
self.assertEqual(data[u'foo'].assible_pos, ('myfile.yml', 2, 6))
# Initial indent (See: #6348)
stream = StringIO(u""" - foo: bar\n baz: qux""")
loader = AssibleLoader(stream, 'myfile.yml')
data = loader.get_single_data()
self.assertEqual(data, [{u'foo': u'bar', u'baz': u'qux'}])
self.assertEqual(data.assible_pos, ('myfile.yml', 1, 2))
self.assertEqual(data[0].assible_pos, ('myfile.yml', 1, 4))
self.assertEqual(data[0][u'foo'].assible_pos, ('myfile.yml', 1, 9))
self.assertEqual(data[0][u'baz'].assible_pos, ('myfile.yml', 2, 9))
def add_fragments(doc, filename, fragment_loader, is_module=False):
fragments = doc.pop('extends_documentation_fragment', [])
if isinstance(fragments, string_types):
fragments = [fragments]
unknown_fragments = []
# doc_fragments are allowed to specify a fragment var other than DOCUMENTATION
# with a . separator; this is complicated by collections-hosted doc_fragments that
# use the same separator. Assume it's collection-hosted normally first, try to load
# as-specified. If failure, assume the right-most component is a var, split it off,
# and retry the load.
for fragment_slug in fragments:
fragment_name = fragment_slug
fragment_var = 'DOCUMENTATION'
fragment_class = fragment_loader.get(fragment_name)
if fragment_class is None and '.' in fragment_slug:
splitname = fragment_slug.rsplit('.', 1)
fragment_name = splitname[0]
fragment_var = splitname[1].upper()
fragment_class = fragment_loader.get(fragment_name)
if fragment_class is None:
unknown_fragments.append(fragment_slug)
continue
fragment_yaml = getattr(fragment_class, fragment_var, None)
if fragment_yaml is None:
if fragment_var != 'DOCUMENTATION':
# if it's asking for something specific that's missing, that's an error
unknown_fragments.append(fragment_slug)
continue
else:
fragment_yaml = '{}' # TODO: this is still an error later since we require 'options' below...
fragment = AssibleLoader(fragment_yaml,
file_name=filename).get_single_data()
real_collection_name = 'assible.builtin'
real_fragment_name = getattr(fragment_class, '_load_name')
if real_fragment_name.startswith('assible_collections.'):
real_collection_name = '.'.join(real_fragment_name.split('.')[1:3])
add_collection_to_versions_and_dates(fragment,
real_collection_name,
is_module=is_module)
if 'notes' in fragment:
notes = fragment.pop('notes')
if notes:
if 'notes' not in doc:
doc['notes'] = []
doc['notes'].extend(notes)
if 'seealso' in fragment:
seealso = fragment.pop('seealso')
if seealso:
if 'seealso' not in doc:
doc['seealso'] = []
doc['seealso'].extend(seealso)
if 'options' not in fragment:
raise Exception(
"missing options in fragment (%s), possibly misformatted?: %s"
% (fragment_name, filename))
# ensure options themselves are directly merged
if 'options' in doc:
try:
merge_fragment(doc['options'], fragment.pop('options'))
except Exception as e:
raise AssibleError("%s options (%s) of unknown type: %s" %
(to_native(e), fragment_name, filename))
else:
doc['options'] = fragment.pop('options')
# merge rest of the sections
try:
merge_fragment(doc, fragment)
except Exception as e:
raise AssibleError("%s (%s) of unknown type: %s" %
(to_native(e), fragment_name, filename))
if unknown_fragments:
raise AssibleError('unknown doc_fragment(s) in file {0}: {1}'.format(
filename, to_native(', '.join(unknown_fragments))))
class TestAssibleLoaderPlay(unittest.TestCase):
def setUp(self):
stream = NameStringIO(u"""
- hosts: localhost
vars:
number: 1
string: Assible
utf8_string: Cafè Eñyei
dictionary:
webster: daniel
oed: oxford
list:
- a
- b
- 1
- 2
tasks:
- name: Test case
ping:
data: "{{ utf8_string }}"
- name: Test 2
ping:
data: "Cafè Eñyei"
- name: Test 3
command: "printf 'Cafè Eñyei\\n'"
""")
self.play_filename = '/path/to/myplay.yml'
stream.name = self.play_filename
self.loader = AssibleLoader(stream)
self.data = self.loader.get_single_data()
def tearDown(self):
pass
def test_data_complete(self):
self.assertEqual(len(self.data), 1)
self.assertIsInstance(self.data, list)
self.assertEqual(frozenset(self.data[0].keys()), frozenset((u'hosts', u'vars', u'tasks')))
self.assertEqual(self.data[0][u'hosts'], u'localhost')
self.assertEqual(self.data[0][u'vars'][u'number'], 1)
self.assertEqual(self.data[0][u'vars'][u'string'], u'Assible')
self.assertEqual(self.data[0][u'vars'][u'utf8_string'], u'Cafè Eñyei')
self.assertEqual(self.data[0][u'vars'][u'dictionary'], {
u'webster': u'daniel',
u'oed': u'oxford'
})
self.assertEqual(self.data[0][u'vars'][u'list'], [u'a', u'b', 1, 2])
self.assertEqual(self.data[0][u'tasks'], [
{u'name': u'Test case', u'ping': {u'data': u'{{ utf8_string }}'}},
{u'name': u'Test 2', u'ping': {u'data': u'Cafè Eñyei'}},
{u'name': u'Test 3', u'command': u'printf \'Cafè Eñyei\n\''},
])
def walk(self, data):
# Make sure there's no str in the data
self.assertNotIsInstance(data, binary_type)
# Descend into various container types
if isinstance(data, text_type):
# strings are a sequence so we have to be explicit here
return
elif isinstance(data, (Sequence, Set)):
for element in data:
self.walk(element)
elif isinstance(data, Mapping):
for k, v in data.items():
self.walk(k)
self.walk(v)
# Scalars were all checked so we're good to go
return
def test_no_str_in_data(self):
# Checks that no strings are str type
self.walk(self.data)
def check_vars(self):
# Numbers don't have line/col information yet
# self.assertEqual(self.data[0][u'vars'][u'number'].assible_pos, (self.play_filename, 4, 21))
self.assertEqual(self.data[0][u'vars'][u'string'].assible_pos, (self.play_filename, 5, 29))
self.assertEqual(self.data[0][u'vars'][u'utf8_string'].assible_pos, (self.play_filename, 6, 34))
self.assertEqual(self.data[0][u'vars'][u'dictionary'].assible_pos, (self.play_filename, 8, 23))
self.assertEqual(self.data[0][u'vars'][u'dictionary'][u'webster'].assible_pos, (self.play_filename, 8, 32))
self.assertEqual(self.data[0][u'vars'][u'dictionary'][u'oed'].assible_pos, (self.play_filename, 9, 28))
self.assertEqual(self.data[0][u'vars'][u'list'].assible_pos, (self.play_filename, 11, 23))
self.assertEqual(self.data[0][u'vars'][u'list'][0].assible_pos, (self.play_filename, 11, 25))
self.assertEqual(self.data[0][u'vars'][u'list'][1].assible_pos, (self.play_filename, 12, 25))
# Numbers don't have line/col info yet
# self.assertEqual(self.data[0][u'vars'][u'list'][2].assible_pos, (self.play_filename, 13, 25))
# self.assertEqual(self.data[0][u'vars'][u'list'][3].assible_pos, (self.play_filename, 14, 25))
def check_tasks(self):
#
# First Task
#
self.assertEqual(self.data[0][u'tasks'][0].assible_pos, (self.play_filename, 16, 23))
self.assertEqual(self.data[0][u'tasks'][0][u'name'].assible_pos, (self.play_filename, 16, 29))
self.assertEqual(self.data[0][u'tasks'][0][u'ping'].assible_pos, (self.play_filename, 18, 25))
self.assertEqual(self.data[0][u'tasks'][0][u'ping'][u'data'].assible_pos, (self.play_filename, 18, 31))
#
# Second Task
#
self.assertEqual(self.data[0][u'tasks'][1].assible_pos, (self.play_filename, 20, 23))
self.assertEqual(self.data[0][u'tasks'][1][u'name'].assible_pos, (self.play_filename, 20, 29))
self.assertEqual(self.data[0][u'tasks'][1][u'ping'].assible_pos, (self.play_filename, 22, 25))
self.assertEqual(self.data[0][u'tasks'][1][u'ping'][u'data'].assible_pos, (self.play_filename, 22, 31))
#
# Third Task
#
self.assertEqual(self.data[0][u'tasks'][2].assible_pos, (self.play_filename, 24, 23))
self.assertEqual(self.data[0][u'tasks'][2][u'name'].assible_pos, (self.play_filename, 24, 29))
self.assertEqual(self.data[0][u'tasks'][2][u'command'].assible_pos, (self.play_filename, 25, 32))
def test_line_numbers(self):
# Check the line/column numbers are correct
# Note: Remember, currently dicts begin at the start of their first entry
self.assertEqual(self.data[0].assible_pos, (self.play_filename, 2, 19))
self.assertEqual(self.data[0][u'hosts'].assible_pos, (self.play_filename, 2, 26))
self.assertEqual(self.data[0][u'vars'].assible_pos, (self.play_filename, 4, 21))
self.check_vars()
self.assertEqual(self.data[0][u'tasks'].assible_pos, (self.play_filename, 16, 21))
self.check_tasks()
def _loader(self, stream):
return AssibleLoader(stream, vault_secrets=self.vault.secrets)
def test_tab_error(self):
stream = StringIO(u"""---\nhosts: localhost\nvars:\n foo: bar\n\tblip: baz""")
loader = AssibleLoader(stream, 'myfile.yml')
self.assertRaises(ScannerError, loader.get_single_data)
def test_error_conditions(self):
stream = StringIO(u"""{""")
loader = AssibleLoader(stream, 'myfile.yml')
self.assertRaises(ParserError, loader.get_single_data)
def _loader(self, stream):
"""Vault related tests will want to override this.
Vault cases should setup a AssibleLoader that has the vault password."""
return AssibleLoader(stream)