def setUp(self): stream = NameStringIO(u""" - hosts: localhost vars: number: 1 string: Assible utf8_string: Cafè Eñyei dictionary: webster: daniel oed: oxford list: - a - b - 1 - 2 tasks: - name: Test case ping: data: "{{ utf8_string }}" - name: Test 2 ping: data: "Cafè Eñyei" - name: Test 3 command: "printf 'Cafè Eñyei\\n'" """) self.play_filename = '/path/to/myplay.yml' stream.name = self.play_filename self.loader = AssibleLoader(stream) self.data = self.loader.get_single_data()
def test_parse_number(self): stream = StringIO(u""" 1 """) loader = AssibleLoader(stream, 'myfile.yml') data = loader.get_single_data() self.assertEqual(data, 1)
def test_parse_utf8_string(self): stream = StringIO(u""" Cafè Eñyei """) loader = AssibleLoader(stream, 'myfile.yml') data = loader.get_single_data() self.assertEqual(data, u'Cafè Eñyei') self.assertIsInstance(data, text_type) self.assertEqual(data.assible_pos, ('myfile.yml', 2, 17))
def _safe_load(stream, file_name=None, vault_secrets=None): ''' Implements yaml.safe_load(), except using our custom loader class. ''' loader = AssibleLoader(stream, file_name, vault_secrets) try: return loader.get_single_data() finally: try: loader.dispose() except AttributeError: pass # older versions of yaml don't have dispose function, ignore
def test_parse_list(self): stream = StringIO(u""" - a - b """) loader = AssibleLoader(stream, 'myfile.yml') data = loader.get_single_data() self.assertEqual(data, [u'a', u'b']) self.assertEqual(len(data), 2) self.assertIsInstance(data[0], text_type) self.assertEqual(data.assible_pos, ('myfile.yml', 2, 17)) self.assertEqual(data[0].assible_pos, ('myfile.yml', 2, 19)) self.assertEqual(data[1].assible_pos, ('myfile.yml', 3, 19))
def test_parse_dict(self): stream = StringIO(u""" webster: daniel oed: oxford """) loader = AssibleLoader(stream, 'myfile.yml') data = loader.get_single_data() self.assertEqual(data, {'webster': 'daniel', 'oed': 'oxford'}) self.assertEqual(len(data), 2) self.assertIsInstance(list(data.keys())[0], text_type) self.assertIsInstance(list(data.values())[0], text_type) # Beginning of the first key self.assertEqual(data.assible_pos, ('myfile.yml', 2, 17)) self.assertEqual(data[u'webster'].assible_pos, ('myfile.yml', 2, 26)) self.assertEqual(data[u'oed'].assible_pos, ('myfile.yml', 3, 22))
def read_docstring(filename, verbose=True, ignore_errors=True): """ Search for assignment of the DOCUMENTATION and EXAMPLES variables in the given file. Parse DOCUMENTATION from YAML and return the YAML doc or None together with EXAMPLES, as plain text. """ data = { 'doc': None, 'plainexamples': None, 'returndocs': None, 'metadata': None, # NOTE: not used anymore, kept for compat 'seealso': None, } string_to_vars = { 'DOCUMENTATION': 'doc', 'EXAMPLES': 'plainexamples', 'RETURN': 'returndocs', 'ASSIBLE_METADATA': 'metadata', # NOTE: now unused, but kept for backwards compat } try: with open(filename, 'rb') as b_module_data: M = ast.parse(b_module_data.read()) for child in M.body: if isinstance(child, ast.Assign): for t in child.targets: try: theid = t.id except AttributeError: # skip errors can happen when trying to use the normal code display.warning("Failed to assign id for %s on %s, skipping" % (t, filename)) continue if theid in string_to_vars: varkey = string_to_vars[theid] if isinstance(child.value, ast.Dict): data[varkey] = ast.literal_eval(child.value) else: if theid == 'EXAMPLES': # examples 'can' be yaml, but even if so, we dont want to parse as such here # as it can create undesired 'objects' that don't display well as docs. data[varkey] = to_text(child.value.s) else: # string should be yaml if already not a dict data[varkey] = AssibleLoader(child.value.s, file_name=filename).get_single_data() display.debug('assigned: %s' % varkey) except Exception: if verbose: display.error("unable to parse %s" % filename) if not ignore_errors: raise return data
def test_parse_short_dict(self): stream = StringIO(u"""{"foo": "bar"}""") loader = AssibleLoader(stream, 'myfile.yml') data = loader.get_single_data() self.assertEqual(data, dict(foo=u'bar')) self.assertEqual(data.assible_pos, ('myfile.yml', 1, 1)) self.assertEqual(data[u'foo'].assible_pos, ('myfile.yml', 1, 9)) stream = StringIO(u"""foo: bar""") loader = AssibleLoader(stream, 'myfile.yml') data = loader.get_single_data() self.assertEqual(data, dict(foo=u'bar')) self.assertEqual(data.assible_pos, ('myfile.yml', 1, 1)) self.assertEqual(data[u'foo'].assible_pos, ('myfile.yml', 1, 6))
def read_docstub(filename): """ Quickly find short_description using string methods instead of node parsing. This does not return a full set of documentation strings and is intended for operations like assible-doc -l. """ in_documentation = False capturing = False indent_detection = '' doc_stub = [] with open(filename, 'r') as t_module_data: for line in t_module_data: if in_documentation: # start capturing the stub until indentation returns if capturing and line.startswith(indent_detection): doc_stub.append(line) elif capturing and not line.startswith(indent_detection): break elif line.lstrip().startswith('short_description:'): capturing = True # Detect that the short_description continues on the next line if it's indented more # than short_description itself. indent_detection = ' ' * (len(line) - len(line.lstrip()) + 1) doc_stub.append(line) elif line.startswith('DOCUMENTATION') and '=' in line: in_documentation = True short_description = r''.join(doc_stub).strip().rstrip('.') data = AssibleLoader(short_description, file_name=filename).get_single_data() return data
def test_front_matter(self): stream = StringIO(u"""---\nfoo: bar""") loader = AssibleLoader(stream, 'myfile.yml') data = loader.get_single_data() self.assertEqual(data, dict(foo=u'bar')) self.assertEqual(data.assible_pos, ('myfile.yml', 2, 1)) self.assertEqual(data[u'foo'].assible_pos, ('myfile.yml', 2, 6)) # Initial indent (See: #6348) stream = StringIO(u""" - foo: bar\n baz: qux""") loader = AssibleLoader(stream, 'myfile.yml') data = loader.get_single_data() self.assertEqual(data, [{u'foo': u'bar', u'baz': u'qux'}]) self.assertEqual(data.assible_pos, ('myfile.yml', 1, 2)) self.assertEqual(data[0].assible_pos, ('myfile.yml', 1, 4)) self.assertEqual(data[0][u'foo'].assible_pos, ('myfile.yml', 1, 9)) self.assertEqual(data[0][u'baz'].assible_pos, ('myfile.yml', 2, 9))
def add_fragments(doc, filename, fragment_loader, is_module=False): fragments = doc.pop('extends_documentation_fragment', []) if isinstance(fragments, string_types): fragments = [fragments] unknown_fragments = [] # doc_fragments are allowed to specify a fragment var other than DOCUMENTATION # with a . separator; this is complicated by collections-hosted doc_fragments that # use the same separator. Assume it's collection-hosted normally first, try to load # as-specified. If failure, assume the right-most component is a var, split it off, # and retry the load. for fragment_slug in fragments: fragment_name = fragment_slug fragment_var = 'DOCUMENTATION' fragment_class = fragment_loader.get(fragment_name) if fragment_class is None and '.' in fragment_slug: splitname = fragment_slug.rsplit('.', 1) fragment_name = splitname[0] fragment_var = splitname[1].upper() fragment_class = fragment_loader.get(fragment_name) if fragment_class is None: unknown_fragments.append(fragment_slug) continue fragment_yaml = getattr(fragment_class, fragment_var, None) if fragment_yaml is None: if fragment_var != 'DOCUMENTATION': # if it's asking for something specific that's missing, that's an error unknown_fragments.append(fragment_slug) continue else: fragment_yaml = '{}' # TODO: this is still an error later since we require 'options' below... fragment = AssibleLoader(fragment_yaml, file_name=filename).get_single_data() real_collection_name = 'assible.builtin' real_fragment_name = getattr(fragment_class, '_load_name') if real_fragment_name.startswith('assible_collections.'): real_collection_name = '.'.join(real_fragment_name.split('.')[1:3]) add_collection_to_versions_and_dates(fragment, real_collection_name, is_module=is_module) if 'notes' in fragment: notes = fragment.pop('notes') if notes: if 'notes' not in doc: doc['notes'] = [] doc['notes'].extend(notes) if 'seealso' in fragment: seealso = fragment.pop('seealso') if seealso: if 'seealso' not in doc: doc['seealso'] = [] doc['seealso'].extend(seealso) if 'options' not in fragment: raise Exception( "missing options in fragment (%s), possibly misformatted?: %s" % (fragment_name, filename)) # ensure options themselves are directly merged if 'options' in doc: try: merge_fragment(doc['options'], fragment.pop('options')) except Exception as e: raise AssibleError("%s options (%s) of unknown type: %s" % (to_native(e), fragment_name, filename)) else: doc['options'] = fragment.pop('options') # merge rest of the sections try: merge_fragment(doc, fragment) except Exception as e: raise AssibleError("%s (%s) of unknown type: %s" % (to_native(e), fragment_name, filename)) if unknown_fragments: raise AssibleError('unknown doc_fragment(s) in file {0}: {1}'.format( filename, to_native(', '.join(unknown_fragments))))
class TestAssibleLoaderPlay(unittest.TestCase): def setUp(self): stream = NameStringIO(u""" - hosts: localhost vars: number: 1 string: Assible utf8_string: Cafè Eñyei dictionary: webster: daniel oed: oxford list: - a - b - 1 - 2 tasks: - name: Test case ping: data: "{{ utf8_string }}" - name: Test 2 ping: data: "Cafè Eñyei" - name: Test 3 command: "printf 'Cafè Eñyei\\n'" """) self.play_filename = '/path/to/myplay.yml' stream.name = self.play_filename self.loader = AssibleLoader(stream) self.data = self.loader.get_single_data() def tearDown(self): pass def test_data_complete(self): self.assertEqual(len(self.data), 1) self.assertIsInstance(self.data, list) self.assertEqual(frozenset(self.data[0].keys()), frozenset((u'hosts', u'vars', u'tasks'))) self.assertEqual(self.data[0][u'hosts'], u'localhost') self.assertEqual(self.data[0][u'vars'][u'number'], 1) self.assertEqual(self.data[0][u'vars'][u'string'], u'Assible') self.assertEqual(self.data[0][u'vars'][u'utf8_string'], u'Cafè Eñyei') self.assertEqual(self.data[0][u'vars'][u'dictionary'], { u'webster': u'daniel', u'oed': u'oxford' }) self.assertEqual(self.data[0][u'vars'][u'list'], [u'a', u'b', 1, 2]) self.assertEqual(self.data[0][u'tasks'], [ {u'name': u'Test case', u'ping': {u'data': u'{{ utf8_string }}'}}, {u'name': u'Test 2', u'ping': {u'data': u'Cafè Eñyei'}}, {u'name': u'Test 3', u'command': u'printf \'Cafè Eñyei\n\''}, ]) def walk(self, data): # Make sure there's no str in the data self.assertNotIsInstance(data, binary_type) # Descend into various container types if isinstance(data, text_type): # strings are a sequence so we have to be explicit here return elif isinstance(data, (Sequence, Set)): for element in data: self.walk(element) elif isinstance(data, Mapping): for k, v in data.items(): self.walk(k) self.walk(v) # Scalars were all checked so we're good to go return def test_no_str_in_data(self): # Checks that no strings are str type self.walk(self.data) def check_vars(self): # Numbers don't have line/col information yet # self.assertEqual(self.data[0][u'vars'][u'number'].assible_pos, (self.play_filename, 4, 21)) self.assertEqual(self.data[0][u'vars'][u'string'].assible_pos, (self.play_filename, 5, 29)) self.assertEqual(self.data[0][u'vars'][u'utf8_string'].assible_pos, (self.play_filename, 6, 34)) self.assertEqual(self.data[0][u'vars'][u'dictionary'].assible_pos, (self.play_filename, 8, 23)) self.assertEqual(self.data[0][u'vars'][u'dictionary'][u'webster'].assible_pos, (self.play_filename, 8, 32)) self.assertEqual(self.data[0][u'vars'][u'dictionary'][u'oed'].assible_pos, (self.play_filename, 9, 28)) self.assertEqual(self.data[0][u'vars'][u'list'].assible_pos, (self.play_filename, 11, 23)) self.assertEqual(self.data[0][u'vars'][u'list'][0].assible_pos, (self.play_filename, 11, 25)) self.assertEqual(self.data[0][u'vars'][u'list'][1].assible_pos, (self.play_filename, 12, 25)) # Numbers don't have line/col info yet # self.assertEqual(self.data[0][u'vars'][u'list'][2].assible_pos, (self.play_filename, 13, 25)) # self.assertEqual(self.data[0][u'vars'][u'list'][3].assible_pos, (self.play_filename, 14, 25)) def check_tasks(self): # # First Task # self.assertEqual(self.data[0][u'tasks'][0].assible_pos, (self.play_filename, 16, 23)) self.assertEqual(self.data[0][u'tasks'][0][u'name'].assible_pos, (self.play_filename, 16, 29)) self.assertEqual(self.data[0][u'tasks'][0][u'ping'].assible_pos, (self.play_filename, 18, 25)) self.assertEqual(self.data[0][u'tasks'][0][u'ping'][u'data'].assible_pos, (self.play_filename, 18, 31)) # # Second Task # self.assertEqual(self.data[0][u'tasks'][1].assible_pos, (self.play_filename, 20, 23)) self.assertEqual(self.data[0][u'tasks'][1][u'name'].assible_pos, (self.play_filename, 20, 29)) self.assertEqual(self.data[0][u'tasks'][1][u'ping'].assible_pos, (self.play_filename, 22, 25)) self.assertEqual(self.data[0][u'tasks'][1][u'ping'][u'data'].assible_pos, (self.play_filename, 22, 31)) # # Third Task # self.assertEqual(self.data[0][u'tasks'][2].assible_pos, (self.play_filename, 24, 23)) self.assertEqual(self.data[0][u'tasks'][2][u'name'].assible_pos, (self.play_filename, 24, 29)) self.assertEqual(self.data[0][u'tasks'][2][u'command'].assible_pos, (self.play_filename, 25, 32)) def test_line_numbers(self): # Check the line/column numbers are correct # Note: Remember, currently dicts begin at the start of their first entry self.assertEqual(self.data[0].assible_pos, (self.play_filename, 2, 19)) self.assertEqual(self.data[0][u'hosts'].assible_pos, (self.play_filename, 2, 26)) self.assertEqual(self.data[0][u'vars'].assible_pos, (self.play_filename, 4, 21)) self.check_vars() self.assertEqual(self.data[0][u'tasks'].assible_pos, (self.play_filename, 16, 21)) self.check_tasks()
def _loader(self, stream): return AssibleLoader(stream, vault_secrets=self.vault.secrets)
def test_tab_error(self): stream = StringIO(u"""---\nhosts: localhost\nvars:\n foo: bar\n\tblip: baz""") loader = AssibleLoader(stream, 'myfile.yml') self.assertRaises(ScannerError, loader.get_single_data)
def test_error_conditions(self): stream = StringIO(u"""{""") loader = AssibleLoader(stream, 'myfile.yml') self.assertRaises(ParserError, loader.get_single_data)
def _loader(self, stream): """Vault related tests will want to override this. Vault cases should setup a AssibleLoader that has the vault password.""" return AssibleLoader(stream)