class TaskResult: ''' This class is responsible for interpretting the resulting data from an executed task, and provides helper methods for determining the result of a given task. ''' def __init__(self, host, task, return_data): self._host = host self._task = task if isinstance(return_data, dict): self._result = return_data.copy() else: self._result = DataLoader().load(return_data) def is_changed(self): return self._check_key('changed') def is_skipped(self): return self._check_key('skipped') def is_failed(self): return self._check_key('failed') or self._result.get('rc', 0) != 0 def is_unreachable(self): return self._check_key('unreachable') def _check_key(self, key): if 'results' in self._result: flag = False for res in self._result.get('results', []): flag |= res.get(key, False) else: return self._result.get(key, False)
class TestDataLoaderWithVault(unittest.TestCase): def setUp(self): self._loader = DataLoader() self._loader.set_vault_password('ansible') def tearDown(self): pass @patch.multiple(DataLoader, path_exists=lambda s, x: True, is_file=lambda s, x: True) def test_parse_from_vault_1_1_file(self): vaulted_data = """$ANSIBLE_VAULT;1.1;AES256 33343734386261666161626433386662623039356366656637303939306563376130623138626165 6436333766346533353463636566313332623130383662340a393835656134633665333861393331 37666233346464636263636530626332623035633135363732623332313534306438393366323966 3135306561356164310a343937653834643433343734653137383339323330626437313562306630 3035 """ if PY3: builtins_name = 'builtins' else: builtins_name = '__builtin__' with patch(builtins_name + '.open', mock_open(read_data=vaulted_data)): output = self._loader.load_from_file('dummy_vault.txt') self.assertEqual(output, dict(foo='bar'))
def __init__(self, host, task, return_data): self._host = host self._task = task if isinstance(return_data, dict): self._result = return_data.copy() else: self._result = DataLoader().load(return_data)
def process_task(body): new_stdin = sys.stdin try: fileno = sys.stdin.fileno() if fileno is not None: try: new_stdin = os.fdopen(os.dup(fileno)) except OSError, e: # couldn't dupe stdin, most likely because it's # not a valid file descriptor, so we just rely on # using the one that was passed in pass except ValueError: # couldn't get stdin's fileno, so we just carry on pass shared_loader_obj = SharedPluginLoaderObj() loader = DataLoader(vault_password=None) task_json = json.loads(body) loader.set_basedir(task_json['base_dir']) host = jsonpickle.decode(task_json['host']) task = jsonpickle.decode(task_json['task']) job_vars = jsonpickle.decode(task_json['task_vars']) connection_info = jsonpickle.decode(task_json['conn_info']) task.set_loader(loader) new_connection_info = connection_info.set_task_and_host_override(task=task, host=host) executor_result = TaskExecutor(host, task, job_vars, new_connection_info, new_stdin, loader, shared_loader_obj).run() task_result = TaskResult(host, task, executor_result) task_pickled = jsonpickle.encode(task_result) return(json.dumps(task_pickled))
class TestDataLoader(unittest.TestCase): def setUp(self): # FIXME: need to add tests that utilize vault_password self._loader = DataLoader() def tearDown(self): pass @patch.object(DataLoader, '_get_file_contents') def test_parse_json_from_file(self, mock_def): mock_def.return_value = ("""{"a": 1, "b": 2, "c": 3}""", True) output = self._loader.load_from_file('dummy_json.txt') self.assertEqual(output, dict(a=1, b=2, c=3)) @patch.object(DataLoader, '_get_file_contents') def test_parse_yaml_from_file(self, mock_def): mock_def.return_value = (""" a: 1 b: 2 c: 3 """, True) output = self._loader.load_from_file('dummy_yaml.txt') self.assertEqual(output, dict(a=1, b=2, c=3)) @patch.object(DataLoader, '_get_file_contents') def test_parse_fail_from_file(self, mock_def): mock_def.return_value = (""" TEXT: *** NOT VALID """, True) self.assertRaises(AnsibleParserError, self._loader.load_from_file, 'dummy_yaml_bad.txt')
class TestDataLoader(unittest.TestCase): def setUp(self): # FIXME: need to add tests that utilize vault_password self._loader = DataLoader() def tearDown(self): pass @patch.object(DataLoader, '_get_file_contents') def test_parse_json_from_file(self, mock_def): mock_def.return_value = ("""{"a": 1, "b": 2, "c": 3}""", True) output = self._loader.load_from_file('dummy_json.txt') self.assertEqual(output, dict(a=1,b=2,c=3)) @patch.object(DataLoader, '_get_file_contents') def test_parse_yaml_from_file(self, mock_def): mock_def.return_value = (""" a: 1 b: 2 c: 3 """, True) output = self._loader.load_from_file('dummy_yaml.txt') self.assertEqual(output, dict(a=1,b=2,c=3)) @patch.object(DataLoader, '_get_file_contents') def test_parse_fail_from_file(self, mock_def): mock_def.return_value = (""" TEXT: *** NOT VALID """, True) self.assertRaises(AnsibleParserError, self._loader.load_from_file, 'dummy_yaml_bad.txt')
def load_data(self, ds, variable_manager=None, loader=None): ''' walk the input datastructure and assign any values ''' assert ds is not None # the variable manager class is used to manage and merge variables # down to a single dictionary for reference in templating, etc. self._variable_manager = variable_manager # the data loader class is used to parse data from strings and files if loader is not None: self._loader = loader else: self._loader = DataLoader() # FIXME: is this required anymore? This doesn't seem to do anything # helpful, and was added in very early stages of the base class # development. #if isinstance(ds, string_types) or isinstance(ds, FileIO): # ds = self._loader.load(ds) # call the preprocess_data() function to massage the data into # something we can more easily parse, and then call the validation # function on it to ensure there are no incorrect key values ds = self.preprocess_data(ds) self._validate_attributes(ds) # Walk all attributes in the class. # # FIXME: we currently don't do anything with private attributes but # may later decide to filter them out of 'ds' here. for name in self._get_base_attributes(): # copy the value over unless a _load_field method is defined if name in ds: method = getattr(self, '_load_%s' % name, None) if method: self._attributes[name] = method(name, ds[name]) else: self._attributes[name] = ds[name] # run early, non-critical validation self.validate() # cache the datastructure internally setattr(self, '_ds', ds) # return the constructed object return self
def run(self): super(VaultCLI, self).run() loader = DataLoader() if self.options.vault_password_file: # read vault_pass from a file self.vault_pass = CLI.read_vault_password_file( self.options.vault_password_file, loader) else: newpass = False rekey = False if self.options.new_vault_password_file: newpass = self.action in ['create', 'rekey', 'encrypt'] rekey = self.action == 'rekey' self.vault_pass, self.new_vault_pass = self.ask_vault_passwords( ask_new_vault_pass=newpass, rekey=rekey) if self.options.new_vault_password_file: # for rekey only self.new_vault_pass = CLI.read_vault_password_file( self.options.new_vault_password_file, loader) if not self.vault_pass: raise AnsibleOptionsError( "A password is required to use Ansible's Vault") self.editor = VaultEditor(self.vault_pass) self.execute()
def run(self): super(VaultCLI, self).run() loader = DataLoader() if self.options.vault_password_file: # read vault_pass from a file self.vault_pass = CLI.read_vault_password_file( self.options.vault_password_file, loader) else: self.vault_pass, _ = self.ask_vault_passwords( ask_vault_pass=True, ask_new_vault_pass=False, confirm_new=False) if self.options.new_vault_password_file: # for rekey only self.new_vault_pass = CLI.read_vault_password_file( self.options.new_vault_password_file, loader) if not self.vault_pass: raise AnsibleOptionsError( "A password is required to use Ansible's Vault") self.editor = VaultEditor(self.vault_pass) self.execute()
class TaskResult: ''' This class is responsible for interpretting the resulting data from an executed task, and provides helper methods for determining the result of a given task. ''' def __init__(self, host, task, return_data): self._host = host self._task = task if isinstance(return_data, dict): self._result = return_data.copy() else: self._result = DataLoader().load(return_data) def is_changed(self): return self._check_key('changed') def is_skipped(self): if 'results' in self._result and self._task.loop: flag = True for res in self._result.get('results', []): if isinstance(res, dict): flag &= res.get('skipped', False) return flag else: return self._result.get('skipped', False) def is_failed(self): if 'failed_when_result' in self._result or \ 'results' in self._result and True in [True for x in self._result['results'] if 'failed_when_result' in x]: return self._check_key('failed_when_result') else: return self._check_key('failed') or self._result.get('rc', 0) != 0 def is_unreachable(self): return self._check_key('unreachable') def _check_key(self, key): if 'results' in self._result and self._task.loop: flag = False for res in self._result.get('results', []): if isinstance(res, dict): flag |= res.get(key, False) return flag else: return self._result.get(key, False)
def load_data(self, ds, variable_manager=None, loader=None): ''' walk the input datastructure and assign any values ''' assert ds is not None # the variable manager class is used to manage and merge variables # down to a single dictionary for reference in templating, etc. self._variable_manager = variable_manager # the data loader class is used to parse data from strings and files if loader is not None: self._loader = loader else: self._loader = DataLoader() # call the preprocess_data() function to massage the data into # something we can more easily parse, and then call the validation # function on it to ensure there are no incorrect key values ds = self.preprocess_data(ds) self._validate_attributes(ds) # Walk all attributes in the class. We sort them based on their priority # so that certain fields can be loaded before others, if they are dependent. # FIXME: we currently don't do anything with private attributes but # may later decide to filter them out of 'ds' here. base_attributes = self._get_base_attributes() for name, attr in sorted(base_attributes.items(), key=operator.itemgetter(1)): # copy the value over unless a _load_field method is defined if name in ds: method = getattr(self, '_load_%s' % name, None) if method: self._attributes[name] = method(name, ds[name]) else: self._attributes[name] = ds[name] # run early, non-critical validation self.validate() # cache the datastructure internally setattr(self, '_ds', ds) # return the constructed object return self
def load_data(self, ds, variable_manager=None, loader=None): ''' walk the input datastructure and assign any values ''' assert ds is not None # the variable manager class is used to manage and merge variables # down to a single dictionary for reference in templating, etc. self._variable_manager = variable_manager # the data loader class is used to parse data from strings and files if loader is not None: self._loader = loader else: self._loader = DataLoader() if isinstance(ds, string_types) or isinstance(ds, FileIO): ds = self._loader.load(ds) # call the munge() function to massage the data into something # we can more easily parse, and then call the validation function # on it to ensure there are no incorrect key values ds = self.munge(ds) self._validate_attributes(ds) # Walk all attributes in the class. # # FIXME: we currently don't do anything with private attributes but # may later decide to filter them out of 'ds' here. for (name, attribute) in iteritems(self._get_base_attributes()): # copy the value over unless a _load_field method is defined if name in ds: method = getattr(self, '_load_%s' % name, None) if method: self._attributes[name] = method(name, ds[name]) else: self._attributes[name] = ds[name] # run early, non-critical validation self.validate() # cache the datastructure internally setattr(self, '_ds', ds) # return the constructed object return self
class Base: def __init__(self): # initialize the data loader and variable manager, which will be provided # later when the object is actually loaded self._loader = None self._variable_manager = None # every object gets a random uuid: self._uuid = uuid.uuid4() # each class knows attributes set upon it, see Task.py for example self._attributes = dict() for (name, value) in iteritems(self._get_base_attributes()): self._attributes[name] = value.default def _get_base_attributes(self): ''' Returns the list of attributes for this class (or any subclass thereof). If the attribute name starts with an underscore, it is removed ''' base_attributes = dict() for (name, value) in getmembers(self.__class__): if isinstance(value, Attribute): if name.startswith('_'): name = name[1:] base_attributes[name] = value return base_attributes def munge(self, ds): ''' infrequently used method to do some pre-processing of legacy terms ''' return ds def load_data(self, ds, variable_manager=None, loader=None): ''' walk the input datastructure and assign any values ''' assert ds is not None # the variable manager class is used to manage and merge variables # down to a single dictionary for reference in templating, etc. self._variable_manager = variable_manager # the data loader class is used to parse data from strings and files if loader is not None: self._loader = loader else: self._loader = DataLoader() if isinstance(ds, string_types) or isinstance(ds, FileIO): ds = self._loader.load(ds) # call the munge() function to massage the data into something # we can more easily parse, and then call the validation function # on it to ensure there are no incorrect key values ds = self.munge(ds) self._validate_attributes(ds) # Walk all attributes in the class. # # FIXME: we currently don't do anything with private attributes but # may later decide to filter them out of 'ds' here. for (name, attribute) in iteritems(self._get_base_attributes()): # copy the value over unless a _load_field method is defined if name in ds: method = getattr(self, '_load_%s' % name, None) if method: self._attributes[name] = method(name, ds[name]) else: self._attributes[name] = ds[name] # run early, non-critical validation self.validate() # cache the datastructure internally setattr(self, '_ds', ds) # return the constructed object return self def get_ds(self): try: return getattr(self, '_ds') except AttributeError: return None def get_loader(self): return self._loader def get_variable_manager(self): return self._variable_manager def _validate_attributes(self, ds): ''' Ensures that there are no keys in the datastructure which do not map to attributes for this object. ''' valid_attrs = [name for (name, attribute) in iteritems(self._get_base_attributes())] for key in ds: if key not in valid_attrs: raise AnsibleParserError("'%s' is not a valid attribute for a %s" % (key, self.__class__.__name__), obj=ds) def validate(self, all_vars=dict()): ''' validation that is done at parse time, not load time ''' # walk all fields in the object for (name, attribute) in iteritems(self._get_base_attributes()): # run validator only if present method = getattr(self, '_validate_%s' % name, None) if method: method(attribute, name, getattr(self, name)) def copy(self): ''' Create a copy of this object and return it. ''' new_me = self.__class__() for (name, attribute) in iteritems(self._get_base_attributes()): setattr(new_me, name, getattr(self, name)) new_me._loader = self._loader new_me._variable_manager = self._variable_manager return new_me def post_validate(self, all_vars=dict(), fail_on_undefined=True): ''' we can't tell that everything is of the right type until we have all the variables. Run basic types (from isa) as well as any _post_validate_<foo> functions. ''' basedir = None if self._loader is not None: basedir = self._loader.get_basedir() templar = Templar(loader=self._loader, variables=all_vars, fail_on_undefined=fail_on_undefined) for (name, attribute) in iteritems(self._get_base_attributes()): if getattr(self, name) is None: if not attribute.required: continue else: raise AnsibleParserError("the field '%s' is required but was not set" % name) try: # if the attribute contains a variable, template it now value = templar.template(getattr(self, name)) # run the post-validator if present method = getattr(self, '_post_validate_%s' % name, None) if method: value = method(attribute, value, all_vars, fail_on_undefined) else: # otherwise, just make sure the attribute is of the type it should be if attribute.isa == 'string': value = unicode(value) elif attribute.isa == 'int': value = int(value) elif attribute.isa == 'bool': value = boolean(value) elif attribute.isa == 'list': if not isinstance(value, list): value = [ value ] elif attribute.isa == 'dict' and not isinstance(value, dict): raise TypeError() # and assign the massaged value back to the attribute field setattr(self, name, value) except (TypeError, ValueError), e: raise AnsibleParserError("the field '%s' has an invalid value (%s), and could not be converted to an %s. Error was: %s" % (name, value, attribute.isa, e), obj=self.get_ds()) except UndefinedError, e: if fail_on_undefined: raise AnsibleParserError("the field '%s' has an invalid value, which appears to include a variable that is undefined. The error was: %s" % (name,e), obj=self.get_ds())
task_result = TaskResult(host, task, executor_result) res_q.put(task_result) else: time.sleep(0.01) except Queue.Empty: pass except (IOError, EOFError, KeyboardInterrupt) as e: debug("got a breaking error: %s" % e) break except Exception as e: debug("EXCEPTION DURING WORKER PROCESSING: %s" % e) traceback.print_exc() break loader = DataLoader() workers = [] for i in range(NUM_WORKERS): main_q = multiprocessing.Queue() res_q = multiprocessing.Queue() worker_p = multiprocessing.Process(target=worker, args=(main_q, res_q, loader)) worker_p.start() workers.append((worker_p, main_q, res_q)) res_q = multiprocessing.Queue() res_p = multiprocessing.Process(target=results, args=(res_q, workers)) res_p.start()
def setUp(self): self._loader = DataLoader(vault_password='******')
def run(self): # Note: slightly wrong, this is written so that implicit localhost # Manage passwords sshpass = None becomepass = None vault_pass = None passwords = {} # don't deal with privilege escalation or passwords when we don't need to if not self.options.listhosts and not self.options.listtasks and not self.options.listtags and not self.options.syntax: self.normalize_become_options() (sshpass, becomepass) = self.ask_passwords() passwords = {'conn_pass': sshpass, 'become_pass': becomepass} if self.options.vault_password_file: # read vault_pass from a file vault_pass = read_vault_file(self.options.vault_password_file) elif self.options.ask_vault_pass: vault_pass = self.ask_vault_passwords(ask_vault_pass=True, ask_new_vault_pass=False, confirm_new=False)[0] loader = DataLoader(vault_password=vault_pass) extra_vars = {} for extra_vars_opt in self.options.extra_vars: extra_vars_opt = to_unicode(extra_vars_opt, errors='strict') if extra_vars_opt.startswith(u"@"): # Argument is a YAML file (JSON is a subset of YAML) data = loader.load_from_file(extra_vars_opt[1:]) elif extra_vars_opt and extra_vars_opt[0] in u'[{': # Arguments as YAML data = loader.load(extra_vars_opt) else: # Arguments as Key-value data = parse_kv(extra_vars_opt) extra_vars = combine_vars(extra_vars, data) # FIXME: this should be moved inside the playbook executor code only_tags = self.options.tags.split(",") skip_tags = self.options.skip_tags if self.options.skip_tags is not None: skip_tags = self.options.skip_tags.split(",") # initial error check, to make sure all specified playbooks are accessible # before we start running anything through the playbook executor for playbook in self.args: if not os.path.exists(playbook): raise AnsibleError("the playbook: %s could not be found" % playbook) if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)): raise AnsibleError( "the playbook: %s does not appear to be a file" % playbook) # create the variable manager, which will be shared throughout # the code, ensuring a consistent view of global variables variable_manager = VariableManager() variable_manager.extra_vars = extra_vars # create the inventory, and filter it based on the subset specified (if any) inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory) variable_manager.set_inventory(inventory) # (which is not returned in list_hosts()) is taken into account for # warning if inventory is empty. But it can't be taken into account for # checking if limit doesn't match any hosts. Instead we don't worry about # limit if only implicit localhost was in inventory to start with. # # Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts()) no_hosts = False if len(inventory.list_hosts()) == 0: # Empty inventory self.display.warning( "provided hosts list is empty, only localhost is available") no_hosts = True inventory.subset(self.options.subset) if len(inventory.list_hosts()) == 0 and no_hosts is False: # Invalid limit raise AnsibleError("Specified --limit does not match any hosts") # create the playbook executor, which manages running the plays via a task queue manager pbex = PlaybookExecutor(playbooks=self.args, inventory=inventory, variable_manager=variable_manager, loader=loader, display=self.display, options=self.options, passwords=passwords) results = pbex.run() if isinstance(results, list): for p in results: self.display.display('\nplaybook: %s\n' % p['playbook']) for play in p['plays']: if self.options.listhosts: self.display.display("\n %s (%s): host count=%d" % (play['name'], play['pattern'], len(play['hosts']))) for host in play['hosts']: self.display.display(" %s" % host) if self.options.listtasks: #TODO: do we want to display block info? self.display.display("\n %s" % (play['name'])) for task in play['tasks']: self.display.display(" %s" % task) if self.options.listtags: #TODO: fix once we figure out block handling above self.display.display("\n %s: tags count=%d" % (play['name'], len(play['tags']))) for tag in play['tags']: self.display.display(" %s" % tag) return 0 else: return results
def run(self): super(PlaybookCLI, self).run() # Note: slightly wrong, this is written so that implicit localhost # Manage passwords sshpass = None becomepass = None vault_pass = None passwords = {} # don't deal with privilege escalation or passwords when we don't need to if not self.options.listhosts and not self.options.listtasks and not self.options.listtags and not self.options.syntax: self.normalize_become_options() (sshpass, becomepass) = self.ask_passwords() passwords = { 'conn_pass': sshpass, 'become_pass': becomepass } if self.options.vault_password_file: # read vault_pass from a file vault_pass = CLI.read_vault_password_file(self.options.vault_password_file) elif self.options.ask_vault_pass: vault_pass = self.ask_vault_passwords(ask_vault_pass=True, ask_new_vault_pass=False, confirm_new=False)[0] loader = DataLoader(vault_password=vault_pass) # initial error check, to make sure all specified playbooks are accessible # before we start running anything through the playbook executor for playbook in self.args: if not os.path.exists(playbook): raise AnsibleError("the playbook: %s could not be found" % playbook) if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)): raise AnsibleError("the playbook: %s does not appear to be a file" % playbook) # create the variable manager, which will be shared throughout # the code, ensuring a consistent view of global variables variable_manager = VariableManager() variable_manager.extra_vars = load_extra_vars(loader=loader, options=self.options) # create the inventory, and filter it based on the subset specified (if any) inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory) variable_manager.set_inventory(inventory) # (which is not returned in list_hosts()) is taken into account for # warning if inventory is empty. But it can't be taken into account for # checking if limit doesn't match any hosts. Instead we don't worry about # limit if only implicit localhost was in inventory to start with. # # Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts()) no_hosts = False if len(inventory.list_hosts()) == 0: # Empty inventory self.display.warning("provided hosts list is empty, only localhost is available") no_hosts = True inventory.subset(self.options.subset) if len(inventory.list_hosts()) == 0 and no_hosts is False: # Invalid limit raise AnsibleError("Specified --limit does not match any hosts") # create the playbook executor, which manages running the plays via a task queue manager pbex = PlaybookExecutor(playbooks=self.args, inventory=inventory, variable_manager=variable_manager, loader=loader, display=self.display, options=self.options, passwords=passwords) results = pbex.run() if isinstance(results, list): for p in results: self.display.display('\nplaybook: %s' % p['playbook']) i = 1 for play in p['plays']: if play.name: playname = play.name else: playname = '#' + str(i) msg = "\n PLAY: %s" % (playname) mytags = set() if self.options.listtags and play.tags: mytags = mytags.union(set(play.tags)) msg += ' TAGS: [%s]' % (','.join(mytags)) if self.options.listhosts: playhosts = set(inventory.get_hosts(play.hosts)) msg += "\n pattern: %s\n hosts (%d):" % (play.hosts, len(playhosts)) for host in playhosts: msg += "\n %s" % host self.display.display(msg) if self.options.listtags or self.options.listtasks: taskmsg = ' tasks:' for block in play.compile(): if not block.has_tasks(): continue j = 1 for task in block.block: taskmsg += "\n %s" % task if self.options.listtags and task.tags: taskmsg += " TAGS: [%s]" % ','.join(mytags.union(set(task.tags))) j = j + 1 self.display.display(taskmsg) i = i + 1 return 0 else: return results
class Base: # connection/transport _connection = FieldAttribute(isa="string") _port = FieldAttribute(isa="int") _remote_user = FieldAttribute(isa="string") # vars and flags _vars = FieldAttribute(isa="dict", default=dict()) _environment = FieldAttribute(isa="dict", default=dict()) _no_log = FieldAttribute(isa="bool", default=False) def __init__(self): # initialize the data loader and variable manager, which will be provided # later when the object is actually loaded self._loader = None self._variable_manager = None # every object gets a random uuid: self._uuid = uuid.uuid4() # and initialize the base attributes self._initialize_base_attributes() # The following three functions are used to programatically define data # descriptors (aka properties) for the Attributes of all of the playbook # objects (tasks, blocks, plays, etc). # # The function signature is a little strange because of how we define # them. We use partial to give each method the name of the Attribute that # it is for. Since partial prefills the positional arguments at the # beginning of the function we end up with the first positional argument # being allocated to the name instead of to the class instance (self) as # normal. To deal with that we make the property name field the first # positional argument and self the second arg. # # Because these methods are defined inside of the class, they get bound to # the instance when the object is created. After we run partial on them # and put the result back into the class as a property, they get bound # a second time. This leads to self being placed in the arguments twice. # To work around that, we mark the functions as @staticmethod so that the # first binding to the instance doesn't happen. @staticmethod def _generic_g(prop_name, self): method = "_get_attr_%s" % prop_name if method in dir(self): return getattr(self, method)() return self._attributes[prop_name] @staticmethod def _generic_s(prop_name, self, value): self._attributes[prop_name] = value @staticmethod def _generic_d(prop_name, self): del self._attributes[prop_name] def _get_base_attributes(self): """ Returns the list of attributes for this class (or any subclass thereof). If the attribute name starts with an underscore, it is removed """ base_attributes = dict() for (name, value) in getmembers(self.__class__): if isinstance(value, Attribute): if name.startswith("_"): name = name[1:] base_attributes[name] = value return base_attributes def _initialize_base_attributes(self): # each class knows attributes set upon it, see Task.py for example self._attributes = dict() for (name, value) in self._get_base_attributes().items(): getter = partial(self._generic_g, name) setter = partial(self._generic_s, name) deleter = partial(self._generic_d, name) # Place the property into the class so that cls.name is the # property functions. setattr(Base, name, property(getter, setter, deleter)) # Place the value into the instance so that the property can # process and hold that value/ setattr(self, name, value.default) def preprocess_data(self, ds): """ infrequently used method to do some pre-processing of legacy terms """ for base_class in self.__class__.mro(): method = getattr(self, "_preprocess_data_%s" % base_class.__name__.lower(), None) if method: return method(ds) return ds def load_data(self, ds, variable_manager=None, loader=None): """ walk the input datastructure and assign any values """ assert ds is not None # the variable manager class is used to manage and merge variables # down to a single dictionary for reference in templating, etc. self._variable_manager = variable_manager # the data loader class is used to parse data from strings and files if loader is not None: self._loader = loader else: self._loader = DataLoader() if isinstance(ds, string_types) or isinstance(ds, FileIO): ds = self._loader.load(ds) # call the preprocess_data() function to massage the data into # something we can more easily parse, and then call the validation # function on it to ensure there are no incorrect key values ds = self.preprocess_data(ds) self._validate_attributes(ds) # Walk all attributes in the class. # # FIXME: we currently don't do anything with private attributes but # may later decide to filter them out of 'ds' here. for name in self._get_base_attributes(): # copy the value over unless a _load_field method is defined if name in ds: method = getattr(self, "_load_%s" % name, None) if method: self._attributes[name] = method(name, ds[name]) else: self._attributes[name] = ds[name] # run early, non-critical validation self.validate() # cache the datastructure internally setattr(self, "_ds", ds) # return the constructed object return self def get_ds(self): try: return getattr(self, "_ds") except AttributeError: return None def get_loader(self): return self._loader def get_variable_manager(self): return self._variable_manager def _validate_attributes(self, ds): """ Ensures that there are no keys in the datastructure which do not map to attributes for this object. """ valid_attrs = frozenset(name for name in self._get_base_attributes()) for key in ds: if key not in valid_attrs: raise AnsibleParserError( "'%s' is not a valid attribute for a %s" % (key, self.__class__.__name__), obj=ds ) def validate(self, all_vars=dict()): """ validation that is done at parse time, not load time """ # walk all fields in the object for (name, attribute) in iteritems(self._get_base_attributes()): # run validator only if present method = getattr(self, "_validate_%s" % name, None) if method: method(attribute, name, getattr(self, name)) def copy(self): """ Create a copy of this object and return it. """ new_me = self.__class__() for name in self._get_base_attributes(): setattr(new_me, name, getattr(self, name)) new_me._loader = self._loader new_me._variable_manager = self._variable_manager # if the ds value was set on the object, copy it to the new copy too if hasattr(self, "_ds"): new_me._ds = self._ds return new_me def post_validate(self, templar): """ we can't tell that everything is of the right type until we have all the variables. Run basic types (from isa) as well as any _post_validate_<foo> functions. """ basedir = None if self._loader is not None: basedir = self._loader.get_basedir() for (name, attribute) in iteritems(self._get_base_attributes()): if getattr(self, name) is None: if not attribute.required: continue else: raise AnsibleParserError("the field '%s' is required but was not set" % name) try: # Run the post-validator if present. These methods are responsible for # using the given templar to template the values, if required. method = getattr(self, "_post_validate_%s" % name, None) if method: value = method(attribute, getattr(self, name), templar) else: # if the attribute contains a variable, template it now value = templar.template(getattr(self, name)) # and make sure the attribute is of the type it should be if value is not None: if attribute.isa == "string": value = unicode(value) elif attribute.isa == "int": value = int(value) elif attribute.isa == "bool": value = boolean(value) elif attribute.isa == "list": if not isinstance(value, list): value = [value] if attribute.listof is not None: for item in value: if not isinstance(item, attribute.listof): raise AnsibleParserError( "the field '%s' should be a list of %s, but the item '%s' is a %s" % (name, attribute.listof, item, type(item)), obj=self.get_ds(), ) elif attribute.isa == "dict" and not isinstance(value, dict): raise TypeError() # and assign the massaged value back to the attribute field setattr(self, name, value) except (TypeError, ValueError) as e: raise AnsibleParserError( "the field '%s' has an invalid value (%s), and could not be converted to an %s. Error was: %s" % (name, value, attribute.isa, e), obj=self.get_ds(), ) except UndefinedError as e: if templar._fail_on_undefined_errors and name != "name": raise AnsibleParserError( "the field '%s' has an invalid value, which appears to include a variable that is undefined. The error was: %s" % (name, e), obj=self.get_ds(), ) def serialize(self): """ Serializes the object derived from the base object into a dictionary of values. This only serializes the field attributes for the object, so this may need to be overridden for any classes which wish to add additional items not stored as field attributes. """ repr = dict() for name in self._get_base_attributes(): repr[name] = getattr(self, name) # serialize the uuid field repr["uuid"] = getattr(self, "_uuid") return repr def deserialize(self, data): """ Given a dictionary of values, load up the field attributes for this object. As with serialize(), if there are any non-field attribute data members, this method will need to be overridden and extended. """ assert isinstance(data, dict) for (name, attribute) in iteritems(self._get_base_attributes()): if name in data: setattr(self, name, data[name]) else: setattr(self, name, attribute.default) # restore the UUID field setattr(self, "_uuid", data.get("uuid")) def _extend_value(self, value, new_value): """ Will extend the value given with new_value (and will turn both into lists if they are not so already). The values are run through a set to remove duplicate values. """ if not isinstance(value, list): value = [value] if not isinstance(new_value, list): new_value = [new_value] # return list(set(value + new_value)) return [i for i, _ in itertools.groupby(value + new_value)] def __getstate__(self): return self.serialize() def __setstate__(self, data): self.__init__() self.deserialize(data)
def run(self): ''' use Runner lib to do SSH things ''' # only thing left should be host pattern pattern = self.args[0] # ignore connection password cause we are local if self.options.connection == "local": self.options.ask_pass = False sshpass = None becomepass = None vault_pass = None self.normalize_become_options() (sshpass, becomepass) = self.ask_passwords() passwords = { 'conn_pass': sshpass, 'become_pass': becomepass } if self.options.vault_password_file: # read vault_pass from a file vault_pass = read_vault_file(self.options.vault_password_file) elif self.options.ask_vault_pass: vault_pass = self.ask_vault_passwords(ask_vault_pass=True, ask_new_vault_pass=False, confirm_new=False)[0] loader = DataLoader(vault_password=vault_pass) variable_manager = VariableManager() inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory) variable_manager.set_inventory(inventory) hosts = inventory.list_hosts(pattern) if len(hosts) == 0: self.display.warning("provided hosts list is empty, only localhost is available") if self.options.listhosts: for host in hosts: self.display.display(' %s' % host) return 0 if self.options.module_name in C.MODULE_REQUIRE_ARGS and not self.options.module_args: raise AnsibleOptionsError("No argument passed to %s module" % self.options.module_name) #TODO: implement async support #if self.options.seconds: # callbacks.display("background launch...\n\n", color='cyan') # results, poller = runner.run_async(self.options.seconds) # results = self.poll_while_needed(poller) #else: # results = runner.run() # create a pseudo-play to execute the specified module via a single task play_ds = dict( name = "Ansible Ad-Hoc", hosts = pattern, gather_facts = 'no', tasks = [ dict(action=dict(module=self.options.module_name, args=parse_kv(self.options.module_args))), ] ) play = Play().load(play_ds, variable_manager=variable_manager, loader=loader) # now create a task queue manager to execute the play tqm = None try: tqm = TaskQueueManager( inventory=inventory, variable_manager=variable_manager, loader=loader, display=self.display, options=self.options, passwords=passwords, stdout_callback='minimal', ) result = tqm.run(play) finally: if tqm: tqm.cleanup() return result
def run(self): super(PlaybookCLI, self).run() # Note: slightly wrong, this is written so that implicit localhost # Manage passwords sshpass = None becomepass = None vault_pass = None passwords = {} # don't deal with privilege escalation or passwords when we don't need to if ( not self.options.listhosts and not self.options.listtasks and not self.options.listtags and not self.options.syntax ): self.normalize_become_options() (sshpass, becomepass) = self.ask_passwords() passwords = {"conn_pass": sshpass, "become_pass": becomepass} loader = DataLoader() if self.options.vault_password_file: # read vault_pass from a file vault_pass = CLI.read_vault_password_file(self.options.vault_password_file, loader=loader) loader.set_vault_password(vault_pass) elif self.options.ask_vault_pass: vault_pass = self.ask_vault_passwords(ask_vault_pass=True, ask_new_vault_pass=False, confirm_new=False)[0] loader.set_vault_password(vault_pass) # initial error check, to make sure all specified playbooks are accessible # before we start running anything through the playbook executor for playbook in self.args: if not os.path.exists(playbook): raise AnsibleError("the playbook: %s could not be found" % playbook) if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)): raise AnsibleError("the playbook: %s does not appear to be a file" % playbook) # create the variable manager, which will be shared throughout # the code, ensuring a consistent view of global variables variable_manager = VariableManager() variable_manager.extra_vars = load_extra_vars(loader=loader, options=self.options) # create the inventory, and filter it based on the subset specified (if any) inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory) variable_manager.set_inventory(inventory) # (which is not returned in list_hosts()) is taken into account for # warning if inventory is empty. But it can't be taken into account for # checking if limit doesn't match any hosts. Instead we don't worry about # limit if only implicit localhost was in inventory to start with. # # Fix this when we rewrite inventory by making localhost a real host (and thus show up in list_hosts()) no_hosts = False if len(inventory.list_hosts()) == 0: # Empty inventory self.display.warning("provided hosts list is empty, only localhost is available") no_hosts = True inventory.subset(self.options.subset) if len(inventory.list_hosts()) == 0 and no_hosts is False: # Invalid limit raise AnsibleError("Specified --limit does not match any hosts") # create the playbook executor, which manages running the plays via a task queue manager pbex = PlaybookExecutor( playbooks=self.args, inventory=inventory, variable_manager=variable_manager, loader=loader, display=self.display, options=self.options, passwords=passwords, ) results = pbex.run() if isinstance(results, list): for p in results: self.display.display("\nplaybook: %s" % p["playbook"]) i = 1 for play in p["plays"]: if play.name: playname = play.name else: playname = "#" + str(i) msg = "\n PLAY: %s" % (playname) mytags = set() if self.options.listtags and play.tags: mytags = mytags.union(set(play.tags)) msg += " TAGS: [%s]" % (",".join(mytags)) if self.options.listhosts: playhosts = set(inventory.get_hosts(play.hosts)) msg += "\n pattern: %s\n hosts (%d):" % (play.hosts, len(playhosts)) for host in playhosts: msg += "\n %s" % host self.display.display(msg) if self.options.listtags or self.options.listtasks: taskmsg = " tasks:" for block in play.compile(): if not block.has_tasks(): continue j = 1 for task in block.block: taskmsg += "\n %s" % task if self.options.listtags and task.tags: taskmsg += " TAGS: [%s]" % ",".join(mytags.union(set(task.tags))) j = j + 1 self.display.display(taskmsg) i = i + 1 return 0 else: return results
class Base: # connection/transport _connection = FieldAttribute(isa='string') _port = FieldAttribute(isa='int') _remote_user = FieldAttribute(isa='string') # variables _vars = FieldAttribute(isa='dict', default=dict(), priority=100) # flags and misc. settings _environment = FieldAttribute(isa='list') _no_log = FieldAttribute(isa='bool') # param names which have been deprecated/removed DEPRECATED_ATTRIBUTES = [ 'sudo', 'sudo_user', 'sudo_pass', 'sudo_exe', 'sudo_flags', 'su', 'su_user', 'su_pass', 'su_exe', 'su_flags', ] def __init__(self): # initialize the data loader and variable manager, which will be provided # later when the object is actually loaded self._loader = None self._variable_manager = None # every object gets a random uuid: self._uuid = uuid.uuid4() # and initialize the base attributes self._initialize_base_attributes() try: from __main__ import display self._display = display except ImportError: from ansible.utils.display import Display self._display = Display() # The following three functions are used to programatically define data # descriptors (aka properties) for the Attributes of all of the playbook # objects (tasks, blocks, plays, etc). # # The function signature is a little strange because of how we define # them. We use partial to give each method the name of the Attribute that # it is for. Since partial prefills the positional arguments at the # beginning of the function we end up with the first positional argument # being allocated to the name instead of to the class instance (self) as # normal. To deal with that we make the property name field the first # positional argument and self the second arg. # # Because these methods are defined inside of the class, they get bound to # the instance when the object is created. After we run partial on them # and put the result back into the class as a property, they get bound # a second time. This leads to self being placed in the arguments twice. # To work around that, we mark the functions as @staticmethod so that the # first binding to the instance doesn't happen. @staticmethod def _generic_g(prop_name, self): method = "_get_attr_%s" % prop_name if hasattr(self, method): return getattr(self, method)() value = self._attributes[prop_name] if value is None and hasattr(self, '_get_parent_attribute'): value = self._get_parent_attribute(prop_name) return value @staticmethod def _generic_s(prop_name, self, value): self._attributes[prop_name] = value @staticmethod def _generic_d(prop_name, self): del self._attributes[prop_name] def _get_base_attributes(self): ''' Returns the list of attributes for this class (or any subclass thereof). If the attribute name starts with an underscore, it is removed ''' base_attributes = dict() for (name, value) in getmembers(self.__class__): if isinstance(value, Attribute): if name.startswith('_'): name = name[1:] base_attributes[name] = value return base_attributes def _initialize_base_attributes(self): # each class knows attributes set upon it, see Task.py for example self._attributes = dict() for (name, value) in self._get_base_attributes().items(): getter = partial(self._generic_g, name) setter = partial(self._generic_s, name) deleter = partial(self._generic_d, name) # Place the property into the class so that cls.name is the # property functions. setattr(Base, name, property(getter, setter, deleter)) # Place the value into the instance so that the property can # process and hold that value/ setattr(self, name, value.default) def preprocess_data(self, ds): ''' infrequently used method to do some pre-processing of legacy terms ''' for base_class in self.__class__.mro(): method = getattr( self, "_preprocess_data_%s" % base_class.__name__.lower(), None) if method: return method(ds) return ds def load_data(self, ds, variable_manager=None, loader=None): ''' walk the input datastructure and assign any values ''' assert ds is not None # cache the datastructure internally setattr(self, '_ds', ds) # the variable manager class is used to manage and merge variables # down to a single dictionary for reference in templating, etc. self._variable_manager = variable_manager # the data loader class is used to parse data from strings and files if loader is not None: self._loader = loader else: self._loader = DataLoader() # call the preprocess_data() function to massage the data into # something we can more easily parse, and then call the validation # function on it to ensure there are no incorrect key values ds = self.preprocess_data(ds) self._validate_attributes(ds) # Walk all attributes in the class. We sort them based on their priority # so that certain fields can be loaded before others, if they are dependent. base_attributes = self._get_base_attributes() for name, attr in sorted(base_attributes.items(), key=operator.itemgetter(1)): # copy the value over unless a _load_field method is defined if name in ds: method = getattr(self, '_load_%s' % name, None) if method: self._attributes[name] = method(name, ds[name]) else: self._attributes[name] = ds[name] # run early, non-critical validation self.validate() # return the constructed object return self def get_ds(self): try: return getattr(self, '_ds') except AttributeError: return None def get_loader(self): return self._loader def get_variable_manager(self): return self._variable_manager def _validate_attributes(self, ds): ''' Ensures that there are no keys in the datastructure which do not map to attributes for this object. ''' valid_attrs = frozenset(name for name in self._get_base_attributes()) for key in ds: if key not in valid_attrs: raise AnsibleParserError( "'%s' is not a valid attribute for a %s" % (key, self.__class__.__name__), obj=ds) def validate(self, all_vars=dict()): ''' validation that is done at parse time, not load time ''' # walk all fields in the object for (name, attribute) in iteritems(self._get_base_attributes()): # run validator only if present method = getattr(self, '_validate_%s' % name, None) if method: method(attribute, name, getattr(self, name)) else: # and make sure the attribute is of the type it should be value = getattr(self, name) if value is not None: if attribute.isa == 'string' and isinstance( value, (list, dict)): raise AnsibleParserError( "The field '%s' is supposed to be a string type, however the incoming data structure is a %s" % (name, type(value)), obj=self.get_ds()) def copy(self): ''' Create a copy of this object and return it. ''' new_me = self.__class__() for name in self._get_base_attributes(): attr_val = getattr(self, name) if isinstance(attr_val, collections.Sequence): setattr(new_me, name, attr_val[:]) elif isinstance(attr_val, collections.Mapping): setattr(new_me, name, attr_val.copy()) else: setattr(new_me, name, attr_val) new_me._loader = self._loader new_me._variable_manager = self._variable_manager # if the ds value was set on the object, copy it to the new copy too if hasattr(self, '_ds'): new_me._ds = self._ds return new_me def post_validate(self, templar): ''' we can't tell that everything is of the right type until we have all the variables. Run basic types (from isa) as well as any _post_validate_<foo> functions. ''' basedir = None if self._loader is not None: basedir = self._loader.get_basedir() # save the omit value for later checking omit_value = templar._available_variables.get('omit') for (name, attribute) in iteritems(self._get_base_attributes()): if getattr(self, name) is None: if not attribute.required: continue else: raise AnsibleParserError( "the field '%s' is required but was not set" % name) elif not attribute.always_post_validate and self.__class__.__name__ not in ( 'Task', 'Handler', 'PlayContext'): # Intermediate objects like Play() won't have their fields validated by # default, as their values are often inherited by other objects and validated # later, so we don't want them to fail out early continue try: # Run the post-validator if present. These methods are responsible for # using the given templar to template the values, if required. method = getattr(self, '_post_validate_%s' % name, None) if method: value = method(attribute, getattr(self, name), templar) else: # if the attribute contains a variable, template it now value = templar.template(getattr(self, name)) # if this evaluated to the omit value, set the value back to # the default specified in the FieldAttribute and move on if omit_value is not None and value == omit_value: setattr(self, name, attribute.default) continue # and make sure the attribute is of the type it should be if value is not None: if attribute.isa == 'string': value = text_type(value) elif attribute.isa == 'int': value = int(value) elif attribute.isa == 'float': value = float(value) elif attribute.isa == 'bool': value = boolean(value) elif attribute.isa == 'percent': # special value, which may be an integer or float # with an optional '%' at the end if isinstance(value, string_types) and '%' in value: value = value.replace('%', '') value = float(value) elif attribute.isa == 'list': if value is None: value = [] elif not isinstance(value, list): value = [value] if attribute.listof is not None: for item in value: if not isinstance(item, attribute.listof): raise AnsibleParserError( "the field '%s' should be a list of %s, but the item '%s' is a %s" % (name, attribute.listof, item, type(item)), obj=self.get_ds()) elif attribute.required and attribute.listof == string_types: if item is None or item.strip() == "": raise AnsibleParserError( "the field '%s' is required, and cannot have empty values" % (name, ), obj=self.get_ds()) elif attribute.isa == 'set': if value is None: value = set() else: if not isinstance(value, (list, set)): value = [value] if not isinstance(value, set): value = set(value) elif attribute.isa == 'dict': if value is None: value = dict() elif not isinstance(value, dict): raise TypeError("%s is not a dictionary" % value) # and assign the massaged value back to the attribute field setattr(self, name, value) except (TypeError, ValueError) as e: raise AnsibleParserError( "the field '%s' has an invalid value (%s), and could not be converted to an %s. Error was: %s" % (name, value, attribute.isa, e), obj=self.get_ds()) except UndefinedError as e: if templar._fail_on_undefined_errors and name != 'name': raise AnsibleParserError( "the field '%s' has an invalid value, which appears to include a variable that is undefined. The error was: %s" % (name, e), obj=self.get_ds()) def serialize(self): ''' Serializes the object derived from the base object into a dictionary of values. This only serializes the field attributes for the object, so this may need to be overridden for any classes which wish to add additional items not stored as field attributes. ''' repr = dict() for name in self._get_base_attributes(): repr[name] = getattr(self, name) # serialize the uuid field repr['uuid'] = getattr(self, '_uuid') return repr def deserialize(self, data): ''' Given a dictionary of values, load up the field attributes for this object. As with serialize(), if there are any non-field attribute data members, this method will need to be overridden and extended. ''' assert isinstance(data, dict) for (name, attribute) in iteritems(self._get_base_attributes()): if name in data: setattr(self, name, data[name]) else: setattr(self, name, attribute.default) # restore the UUID field setattr(self, '_uuid', data.get('uuid')) def _load_vars(self, attr, ds): ''' Vars in a play can be specified either as a dictionary directly, or as a list of dictionaries. If the later, this method will turn the list into a single dictionary. ''' def _validate_variable_keys(ds): for key in ds: if not isidentifier(key): raise TypeError("%s is not a valid variable name" % key) try: if isinstance(ds, dict): _validate_variable_keys(ds) return ds elif isinstance(ds, list): all_vars = dict() for item in ds: if not isinstance(item, dict): raise ValueError _validate_variable_keys(item) all_vars = combine_vars(all_vars, item) return all_vars elif ds is None: return {} else: raise ValueError except ValueError: raise AnsibleParserError( "Vars in a %s must be specified as a dictionary, or a list of dictionaries" % self.__class__.__name__, obj=ds) except TypeError as e: raise AnsibleParserError( "Invalid variable name in vars specified for %s: %s" % (self.__class__.__name__, e), obj=ds) def _extend_value(self, value, new_value): ''' Will extend the value given with new_value (and will turn both into lists if they are not so already). The values are run through a set to remove duplicate values. ''' if not isinstance(value, list): value = [value] if not isinstance(new_value, list): new_value = [new_value] #return list(set(value + new_value)) return [i for i, _ in itertools.groupby(value + new_value)] def __getstate__(self): return self.serialize() def __setstate__(self, data): self.__init__() self.deserialize(data)
def run(self): ''' use Runner lib to do SSH things ''' super(AdHocCLI, self).run() # only thing left should be host pattern pattern = self.args[0] # ignore connection password cause we are local if self.options.connection == "local": self.options.ask_pass = False sshpass = None becomepass = None vault_pass = None self.normalize_become_options() (sshpass, becomepass) = self.ask_passwords() passwords = { 'conn_pass': sshpass, 'become_pass': becomepass } if self.options.vault_password_file: # read vault_pass from a file vault_pass = CLI.read_vault_password_file(self.options.vault_password_file) elif self.options.ask_vault_pass: vault_pass = self.ask_vault_passwords(ask_vault_pass=True, ask_new_vault_pass=False, confirm_new=False)[0] loader = DataLoader(vault_password=vault_pass) variable_manager = VariableManager() variable_manager.extra_vars = load_extra_vars(loader=loader, options=self.options) inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=self.options.inventory) variable_manager.set_inventory(inventory) hosts = inventory.list_hosts(pattern) if len(hosts) == 0: self.display.warning("provided hosts list is empty, only localhost is available") if self.options.listhosts: self.display.display(' hosts (%d):' % len(hosts)) for host in hosts: self.display.display(' %s' % host) return 0 if self.options.module_name in C.MODULE_REQUIRE_ARGS and not self.options.module_args: err = "No argument passed to %s module" % self.options.module_name if pattern.endswith(".yml"): err = err + ' (did you mean to run ansible-playbook?)' raise AnsibleOptionsError(err) #TODO: implement async support #if self.options.seconds: # callbacks.display("background launch...\n\n", color='cyan') # results, poller = runner.run_async(self.options.seconds) # results = self.poll_while_needed(poller) #else: # results = runner.run() # create a pseudo-play to execute the specified module via a single task play_ds = self._play_ds(pattern) play = Play().load(play_ds, variable_manager=variable_manager, loader=loader) if self.options.one_line: cb = 'oneline' else: cb = 'minimal' # now create a task queue manager to execute the play self._tqm = None try: self._tqm = TaskQueueManager( inventory=inventory, variable_manager=variable_manager, loader=loader, display=self.display, options=self.options, passwords=passwords, stdout_callback=cb, ) result = self._tqm.run(play) finally: if self._tqm: self._tqm.cleanup() return result
def setUp(self): self._loader = DataLoader() self._loader.set_vault_password('ansible')
def setUp(self): self._loader = DataLoader()
class Base: def __init__(self): # initialize the data loader and variable manager, which will be provided # later when the object is actually loaded self._loader = None self._variable_manager = None # every object gets a random uuid: self._uuid = uuid.uuid4() # and initialize the base attributes self._initialize_base_attributes() @staticmethod def _generic_g(key, self): method = "_get_attr_%s" % key if method in dir(self): return getattr(self, method)() return self._attributes[key] @staticmethod def _generic_s(key, self, value): self._attributes[key] = value @staticmethod def _generic_d(key, self): del self._attributes[key] def _get_base_attributes(self): ''' Returns the list of attributes for this class (or any subclass thereof). If the attribute name starts with an underscore, it is removed ''' base_attributes = dict() for (name, value) in getmembers(self.__class__): if isinstance(value, Attribute): if name.startswith('_'): name = name[1:] base_attributes[name] = value return base_attributes def _initialize_base_attributes(self): # each class knows attributes set upon it, see Task.py for example self._attributes = dict() for (name, value) in self._get_base_attributes().items(): getter = partial(self._generic_g, name) setter = partial(self._generic_s, name) deleter = partial(self._generic_d, name) setattr(Base, name, property(getter, setter, deleter)) setattr(self, name, value.default) def munge(self, ds): ''' infrequently used method to do some pre-processing of legacy terms ''' for base_class in self.__class__.mro(): method = getattr(self, "_munge_%s" % base_class.__name__.lower(), None) if method: return method(ds) return ds def load_data(self, ds, variable_manager=None, loader=None): ''' walk the input datastructure and assign any values ''' assert ds is not None # the variable manager class is used to manage and merge variables # down to a single dictionary for reference in templating, etc. self._variable_manager = variable_manager # the data loader class is used to parse data from strings and files if loader is not None: self._loader = loader else: self._loader = DataLoader() if isinstance(ds, string_types) or isinstance(ds, FileIO): ds = self._loader.load(ds) # call the munge() function to massage the data into something # we can more easily parse, and then call the validation function # on it to ensure there are no incorrect key values ds = self.munge(ds) self._validate_attributes(ds) # Walk all attributes in the class. # # FIXME: we currently don't do anything with private attributes but # may later decide to filter them out of 'ds' here. for name in self._get_base_attributes(): # copy the value over unless a _load_field method is defined if name in ds: method = getattr(self, '_load_%s' % name, None) if method: self._attributes[name] = method(name, ds[name]) else: self._attributes[name] = ds[name] # run early, non-critical validation self.validate() # cache the datastructure internally setattr(self, '_ds', ds) # return the constructed object return self def get_ds(self): try: return getattr(self, '_ds') except AttributeError: return None def get_loader(self): return self._loader def get_variable_manager(self): return self._variable_manager def _validate_attributes(self, ds): ''' Ensures that there are no keys in the datastructure which do not map to attributes for this object. ''' valid_attrs = frozenset(name for name in self._get_base_attributes()) for key in ds: if key not in valid_attrs: raise AnsibleParserError( "'%s' is not a valid attribute for a %s" % (key, self.__class__.__name__), obj=ds) def validate(self, all_vars=dict()): ''' validation that is done at parse time, not load time ''' # walk all fields in the object for (name, attribute) in iteritems(self._get_base_attributes()): # run validator only if present method = getattr(self, '_validate_%s' % name, None) if method: method(attribute, name, getattr(self, name)) def copy(self): ''' Create a copy of this object and return it. ''' new_me = self.__class__() for name in self._get_base_attributes(): setattr(new_me, name, getattr(self, name)) new_me._loader = self._loader new_me._variable_manager = self._variable_manager return new_me def post_validate(self, all_vars=dict(), fail_on_undefined=True): ''' we can't tell that everything is of the right type until we have all the variables. Run basic types (from isa) as well as any _post_validate_<foo> functions. ''' basedir = None if self._loader is not None: basedir = self._loader.get_basedir() templar = Templar(loader=self._loader, variables=all_vars, fail_on_undefined=fail_on_undefined) for (name, attribute) in iteritems(self._get_base_attributes()): if getattr(self, name) is None: if not attribute.required: continue else: raise AnsibleParserError( "the field '%s' is required but was not set" % name) try: # if the attribute contains a variable, template it now value = templar.template(getattr(self, name)) # run the post-validator if present method = getattr(self, '_post_validate_%s' % name, None) if method: value = method(attribute, value, all_vars, fail_on_undefined) else: # otherwise, just make sure the attribute is of the type it should be if attribute.isa == 'string': value = unicode(value) elif attribute.isa == 'int': value = int(value) elif attribute.isa == 'bool': value = boolean(value) elif attribute.isa == 'list': if not isinstance(value, list): value = [value] elif attribute.isa == 'dict' and not isinstance( value, dict): raise TypeError() # and assign the massaged value back to the attribute field setattr(self, name, value) except (TypeError, ValueError), e: raise AnsibleParserError( "the field '%s' has an invalid value (%s), and could not be converted to an %s. Error was: %s" % (name, value, attribute.isa, e), obj=self.get_ds()) except UndefinedError, e: if fail_on_undefined: raise AnsibleParserError( "the field '%s' has an invalid value, which appears to include a variable that is undefined. The error was: %s" % (name, e), obj=self.get_ds())
class Base: # connection/transport _connection = FieldAttribute(isa='string') _port = FieldAttribute(isa='int') _remote_user = FieldAttribute(isa='string') # vars and flags _vars = FieldAttribute(isa='dict', default=dict()) _environment = FieldAttribute(isa='dict', default=dict()) _no_log = FieldAttribute(isa='bool', default=False) def __init__(self): # initialize the data loader and variable manager, which will be provided # later when the object is actually loaded self._loader = None self._variable_manager = None # every object gets a random uuid: self._uuid = uuid.uuid4() # and initialize the base attributes self._initialize_base_attributes() # The following three functions are used to programatically define data # descriptors (aka properties) for the Attributes of all of the playbook # objects (tasks, blocks, plays, etc). # # The function signature is a little strange because of how we define # them. We use partial to give each method the name of the Attribute that # it is for. Since partial prefills the positional arguments at the # beginning of the function we end up with the first positional argument # being allocated to the name instead of to the class instance (self) as # normal. To deal with that we make the property name field the first # positional argument and self the second arg. # # Because these methods are defined inside of the class, they get bound to # the instance when the object is created. After we run partial on them # and put the result back into the class as a property, they get bound # a second time. This leads to self being placed in the arguments twice. # To work around that, we mark the functions as @staticmethod so that the # first binding to the instance doesn't happen. @staticmethod def _generic_g(prop_name, self): method = "_get_attr_%s" % prop_name if method in dir(self): return getattr(self, method)() return self._attributes[prop_name] @staticmethod def _generic_s(prop_name, self, value): self._attributes[prop_name] = value @staticmethod def _generic_d(prop_name, self): del self._attributes[prop_name] def _get_base_attributes(self): ''' Returns the list of attributes for this class (or any subclass thereof). If the attribute name starts with an underscore, it is removed ''' base_attributes = dict() for (name, value) in getmembers(self.__class__): if isinstance(value, Attribute): if name.startswith('_'): name = name[1:] base_attributes[name] = value return base_attributes def _initialize_base_attributes(self): # each class knows attributes set upon it, see Task.py for example self._attributes = dict() for (name, value) in self._get_base_attributes().items(): getter = partial(self._generic_g, name) setter = partial(self._generic_s, name) deleter = partial(self._generic_d, name) # Place the property into the class so that cls.name is the # property functions. setattr(Base, name, property(getter, setter, deleter)) # Place the value into the instance so that the property can # process and hold that value/ setattr(self, name, value.default) def preprocess_data(self, ds): ''' infrequently used method to do some pre-processing of legacy terms ''' for base_class in self.__class__.mro(): method = getattr( self, "_preprocess_data_%s" % base_class.__name__.lower(), None) if method: return method(ds) return ds def load_data(self, ds, variable_manager=None, loader=None): ''' walk the input datastructure and assign any values ''' assert ds is not None # the variable manager class is used to manage and merge variables # down to a single dictionary for reference in templating, etc. self._variable_manager = variable_manager # the data loader class is used to parse data from strings and files if loader is not None: self._loader = loader else: self._loader = DataLoader() if isinstance(ds, string_types) or isinstance(ds, FileIO): ds = self._loader.load(ds) # call the preprocess_data() function to massage the data into # something we can more easily parse, and then call the validation # function on it to ensure there are no incorrect key values ds = self.preprocess_data(ds) self._validate_attributes(ds) # Walk all attributes in the class. # # FIXME: we currently don't do anything with private attributes but # may later decide to filter them out of 'ds' here. for name in self._get_base_attributes(): # copy the value over unless a _load_field method is defined if name in ds: method = getattr(self, '_load_%s' % name, None) if method: self._attributes[name] = method(name, ds[name]) else: self._attributes[name] = ds[name] # run early, non-critical validation self.validate() # cache the datastructure internally setattr(self, '_ds', ds) # return the constructed object return self def get_ds(self): try: return getattr(self, '_ds') except AttributeError: return None def get_loader(self): return self._loader def get_variable_manager(self): return self._variable_manager def _validate_attributes(self, ds): ''' Ensures that there are no keys in the datastructure which do not map to attributes for this object. ''' valid_attrs = frozenset(name for name in self._get_base_attributes()) for key in ds: if key not in valid_attrs: raise AnsibleParserError( "'%s' is not a valid attribute for a %s" % (key, self.__class__.__name__), obj=ds) def validate(self, all_vars=dict()): ''' validation that is done at parse time, not load time ''' # walk all fields in the object for (name, attribute) in iteritems(self._get_base_attributes()): # run validator only if present method = getattr(self, '_validate_%s' % name, None) if method: method(attribute, name, getattr(self, name)) def copy(self): ''' Create a copy of this object and return it. ''' new_me = self.__class__() for name in self._get_base_attributes(): setattr(new_me, name, getattr(self, name)) new_me._loader = self._loader new_me._variable_manager = self._variable_manager # if the ds value was set on the object, copy it to the new copy too if hasattr(self, '_ds'): new_me._ds = self._ds return new_me def post_validate(self, templar): ''' we can't tell that everything is of the right type until we have all the variables. Run basic types (from isa) as well as any _post_validate_<foo> functions. ''' basedir = None if self._loader is not None: basedir = self._loader.get_basedir() for (name, attribute) in iteritems(self._get_base_attributes()): if getattr(self, name) is None: if not attribute.required: continue else: raise AnsibleParserError( "the field '%s' is required but was not set" % name) try: # Run the post-validator if present. These methods are responsible for # using the given templar to template the values, if required. method = getattr(self, '_post_validate_%s' % name, None) if method: value = method(attribute, getattr(self, name), templar) else: # if the attribute contains a variable, template it now value = templar.template(getattr(self, name)) # and make sure the attribute is of the type it should be if value is not None: if attribute.isa == 'string': value = unicode(value) elif attribute.isa == 'int': value = int(value) elif attribute.isa == 'bool': value = boolean(value) elif attribute.isa == 'list': if not isinstance(value, list): value = [value] if attribute.listof is not None: for item in value: if not isinstance(item, attribute.listof): raise AnsibleParserError( "the field '%s' should be a list of %s, but the item '%s' is a %s" % (name, attribute.listof, item, type(item)), obj=self.get_ds()) elif attribute.isa == 'dict' and not isinstance( value, dict): raise TypeError() # and assign the massaged value back to the attribute field setattr(self, name, value) except (TypeError, ValueError) as e: raise AnsibleParserError( "the field '%s' has an invalid value (%s), and could not be converted to an %s. Error was: %s" % (name, value, attribute.isa, e), obj=self.get_ds()) except UndefinedError as e: if templar._fail_on_undefined_errors and name != 'name': raise AnsibleParserError( "the field '%s' has an invalid value, which appears to include a variable that is undefined. The error was: %s" % (name, e), obj=self.get_ds()) def serialize(self): ''' Serializes the object derived from the base object into a dictionary of values. This only serializes the field attributes for the object, so this may need to be overridden for any classes which wish to add additional items not stored as field attributes. ''' repr = dict() for name in self._get_base_attributes(): repr[name] = getattr(self, name) # serialize the uuid field repr['uuid'] = getattr(self, '_uuid') return repr def deserialize(self, data): ''' Given a dictionary of values, load up the field attributes for this object. As with serialize(), if there are any non-field attribute data members, this method will need to be overridden and extended. ''' assert isinstance(data, dict) for (name, attribute) in iteritems(self._get_base_attributes()): if name in data: setattr(self, name, data[name]) else: setattr(self, name, attribute.default) # restore the UUID field setattr(self, '_uuid', data.get('uuid')) def _extend_value(self, value, new_value): ''' Will extend the value given with new_value (and will turn both into lists if they are not so already). The values are run through a set to remove duplicate values. ''' if not isinstance(value, list): value = [value] if not isinstance(new_value, list): new_value = [new_value] #return list(set(value + new_value)) return [i for i, _ in itertools.groupby(value + new_value)] def __getstate__(self): return self.serialize() def __setstate__(self, data): self.__init__() self.deserialize(data)
class Base: # connection/transport _connection = FieldAttribute(isa='string') _port = FieldAttribute(isa='int') _remote_user = FieldAttribute(isa='string') # variables _vars = FieldAttribute(isa='dict', default=dict(), priority=100) # flags and misc. settings _environment = FieldAttribute(isa='list') _no_log = FieldAttribute(isa='bool') # param names which have been deprecated/removed DEPRECATED_ATTRIBUTES = [ 'sudo', 'sudo_user', 'sudo_pass', 'sudo_exe', 'sudo_flags', 'su', 'su_user', 'su_pass', 'su_exe', 'su_flags', ] def __init__(self): # initialize the data loader and variable manager, which will be provided # later when the object is actually loaded self._loader = None self._variable_manager = None # every object gets a random uuid: self._uuid = uuid.uuid4() # and initialize the base attributes self._initialize_base_attributes() try: from __main__ import display self._display = display except ImportError: from ansible.utils.display import Display self._display = Display() # The following three functions are used to programatically define data # descriptors (aka properties) for the Attributes of all of the playbook # objects (tasks, blocks, plays, etc). # # The function signature is a little strange because of how we define # them. We use partial to give each method the name of the Attribute that # it is for. Since partial prefills the positional arguments at the # beginning of the function we end up with the first positional argument # being allocated to the name instead of to the class instance (self) as # normal. To deal with that we make the property name field the first # positional argument and self the second arg. # # Because these methods are defined inside of the class, they get bound to # the instance when the object is created. After we run partial on them # and put the result back into the class as a property, they get bound # a second time. This leads to self being placed in the arguments twice. # To work around that, we mark the functions as @staticmethod so that the # first binding to the instance doesn't happen. @staticmethod def _generic_g(prop_name, self): method = "_get_attr_%s" % prop_name if hasattr(self, method): return getattr(self, method)() return self._attributes[prop_name] @staticmethod def _generic_s(prop_name, self, value): self._attributes[prop_name] = value @staticmethod def _generic_d(prop_name, self): del self._attributes[prop_name] def _get_base_attributes(self): ''' Returns the list of attributes for this class (or any subclass thereof). If the attribute name starts with an underscore, it is removed ''' base_attributes = dict() for (name, value) in getmembers(self.__class__): if isinstance(value, Attribute): if name.startswith('_'): name = name[1:] base_attributes[name] = value return base_attributes def _initialize_base_attributes(self): # each class knows attributes set upon it, see Task.py for example self._attributes = dict() for (name, value) in self._get_base_attributes().items(): getter = partial(self._generic_g, name) setter = partial(self._generic_s, name) deleter = partial(self._generic_d, name) # Place the property into the class so that cls.name is the # property functions. setattr(Base, name, property(getter, setter, deleter)) # Place the value into the instance so that the property can # process and hold that value/ setattr(self, name, value.default) def preprocess_data(self, ds): ''' infrequently used method to do some pre-processing of legacy terms ''' for base_class in self.__class__.mro(): method = getattr(self, "_preprocess_data_%s" % base_class.__name__.lower(), None) if method: return method(ds) return ds def load_data(self, ds, variable_manager=None, loader=None): ''' walk the input datastructure and assign any values ''' assert ds is not None # the variable manager class is used to manage and merge variables # down to a single dictionary for reference in templating, etc. self._variable_manager = variable_manager # the data loader class is used to parse data from strings and files if loader is not None: self._loader = loader else: self._loader = DataLoader() # call the preprocess_data() function to massage the data into # something we can more easily parse, and then call the validation # function on it to ensure there are no incorrect key values ds = self.preprocess_data(ds) self._validate_attributes(ds) # Walk all attributes in the class. We sort them based on their priority # so that certain fields can be loaded before others, if they are dependent. # FIXME: we currently don't do anything with private attributes but # may later decide to filter them out of 'ds' here. base_attributes = self._get_base_attributes() for name, attr in sorted(base_attributes.items(), key=operator.itemgetter(1)): # copy the value over unless a _load_field method is defined if name in ds: method = getattr(self, '_load_%s' % name, None) if method: self._attributes[name] = method(name, ds[name]) else: self._attributes[name] = ds[name] # run early, non-critical validation self.validate() # cache the datastructure internally setattr(self, '_ds', ds) # return the constructed object return self def get_ds(self): try: return getattr(self, '_ds') except AttributeError: return None def get_loader(self): return self._loader def get_variable_manager(self): return self._variable_manager def _validate_attributes(self, ds): ''' Ensures that there are no keys in the datastructure which do not map to attributes for this object. ''' valid_attrs = frozenset(name for name in self._get_base_attributes()) for key in ds: if key not in valid_attrs: raise AnsibleParserError("'%s' is not a valid attribute for a %s" % (key, self.__class__.__name__), obj=ds) def validate(self, all_vars=dict()): ''' validation that is done at parse time, not load time ''' # walk all fields in the object for (name, attribute) in iteritems(self._get_base_attributes()): # run validator only if present method = getattr(self, '_validate_%s' % name, None) if method: method(attribute, name, getattr(self, name)) def copy(self): ''' Create a copy of this object and return it. ''' new_me = self.__class__() for name in self._get_base_attributes(): setattr(new_me, name, getattr(self, name)) new_me._loader = self._loader new_me._variable_manager = self._variable_manager # if the ds value was set on the object, copy it to the new copy too if hasattr(self, '_ds'): new_me._ds = self._ds return new_me def post_validate(self, templar): ''' we can't tell that everything is of the right type until we have all the variables. Run basic types (from isa) as well as any _post_validate_<foo> functions. ''' basedir = None if self._loader is not None: basedir = self._loader.get_basedir() # save the omit value for later checking omit_value = templar._available_variables.get('omit') for (name, attribute) in iteritems(self._get_base_attributes()): if getattr(self, name) is None: if not attribute.required: continue else: raise AnsibleParserError("the field '%s' is required but was not set" % name) elif not attribute.always_post_validate and self.__class__.__name__ not in ('Task', 'Handler', 'PlayContext'): # Intermediate objects like Play() won't have their fields validated by # default, as their values are often inherited by other objects and validated # later, so we don't want them to fail out early continue try: # Run the post-validator if present. These methods are responsible for # using the given templar to template the values, if required. method = getattr(self, '_post_validate_%s' % name, None) if method: value = method(attribute, getattr(self, name), templar) else: # if the attribute contains a variable, template it now value = templar.template(getattr(self, name)) # if this evaluated to the omit value, set the value back to # the default specified in the FieldAttribute and move on if omit_value is not None and value == omit_value: value = attribute.default continue # and make sure the attribute is of the type it should be if value is not None: if attribute.isa == 'string': value = text_type(value) elif attribute.isa == 'int': value = int(value) elif attribute.isa == 'float': value = float(value) elif attribute.isa == 'bool': value = boolean(value) elif attribute.isa == 'percent': # special value, which may be an integer or float # with an optional '%' at the end if isinstance(value, string_types) and '%' in value: value = value.replace('%', '') value = float(value) elif attribute.isa == 'list': if value is None: value = [] elif not isinstance(value, list): value = [ value ] if attribute.listof is not None: for item in value: if not isinstance(item, attribute.listof): raise AnsibleParserError("the field '%s' should be a list of %s, but the item '%s' is a %s" % (name, attribute.listof, item, type(item)), obj=self.get_ds()) elif attribute.required and attribute.listof == string_types: if item is None or item.strip() == "": raise AnsibleParserError("the field '%s' is required, and cannot have empty values" % (name,), obj=self.get_ds()) elif attribute.isa == 'set': if value is None: value = set() else: if not isinstance(value, (list, set)): value = [ value ] if not isinstance(value, set): value = set(value) elif attribute.isa == 'dict': if value is None: value = dict() elif not isinstance(value, dict): raise TypeError("%s is not a dictionary" % value) # and assign the massaged value back to the attribute field setattr(self, name, value) except (TypeError, ValueError) as e: raise AnsibleParserError("the field '%s' has an invalid value (%s), and could not be converted to an %s. Error was: %s" % (name, value, attribute.isa, e), obj=self.get_ds()) except UndefinedError as e: if templar._fail_on_undefined_errors and name != 'name': raise AnsibleParserError("the field '%s' has an invalid value, which appears to include a variable that is undefined. The error was: %s" % (name,e), obj=self.get_ds()) def serialize(self): ''' Serializes the object derived from the base object into a dictionary of values. This only serializes the field attributes for the object, so this may need to be overridden for any classes which wish to add additional items not stored as field attributes. ''' repr = dict() for name in self._get_base_attributes(): repr[name] = getattr(self, name) # serialize the uuid field repr['uuid'] = getattr(self, '_uuid') return repr def deserialize(self, data): ''' Given a dictionary of values, load up the field attributes for this object. As with serialize(), if there are any non-field attribute data members, this method will need to be overridden and extended. ''' assert isinstance(data, dict) for (name, attribute) in iteritems(self._get_base_attributes()): if name in data: setattr(self, name, data[name]) else: setattr(self, name, attribute.default) # restore the UUID field setattr(self, '_uuid', data.get('uuid')) def _load_vars(self, attr, ds): ''' Vars in a play can be specified either as a dictionary directly, or as a list of dictionaries. If the later, this method will turn the list into a single dictionary. ''' def _validate_variable_keys(ds): for key in ds: if not isidentifier(key): raise TypeError("%s is not a valid variable name" % key) try: if isinstance(ds, dict): _validate_variable_keys(ds) return ds elif isinstance(ds, list): all_vars = dict() for item in ds: if not isinstance(item, dict): raise ValueError _validate_variable_keys(item) all_vars = combine_vars(all_vars, item) return all_vars elif ds is None: return {} else: raise ValueError except ValueError: raise AnsibleParserError("Vars in a %s must be specified as a dictionary, or a list of dictionaries" % self.__class__.__name__, obj=ds) except TypeError as e: raise AnsibleParserError("Invalid variable name in vars specified for %s: %s" % (self.__class__.__name__, e), obj=ds) def _extend_value(self, value, new_value): ''' Will extend the value given with new_value (and will turn both into lists if they are not so already). The values are run through a set to remove duplicate values. ''' if not isinstance(value, list): value = [ value ] if not isinstance(new_value, list): new_value = [ new_value ] #return list(set(value + new_value)) return [i for i,_ in itertools.groupby(value + new_value)] def __getstate__(self): return self.serialize() def __setstate__(self, data): self.__init__() self.deserialize(data)
def setUp(self): # FIXME: need to add tests that utilize vault_password self._loader = DataLoader()