def validate(self, target, old_schema=False): """ Attemptes to validate a single target in a PinFile Returns the groups returned when the topology is validated :param target: the target being validated :param old_schema: whether or not to validate using the old schema by default """ for field in target.keys(): if field not in Validator.SECTIONS: raise ValidationError("Section '{0}' not a valid top-level " "PinFile section. valid sections " "are '{1}'".format( target, Validator.SECTIONS)) if 'topology' not in list(target.keys()): raise ValidationError("Each target must have a topology") topo_data = target['topology'] try: resources = self.validate_topology(topo_data) except (SchemaError, KeyError): # if topology fails, try converting from old to new style self._convert_topology(topo_data) resources = self.validate_topology(topo_data) if 'layout' in list(target.keys()): self.validate_layout(target['layout']) if 'cfgs' in list(target.keys()): self.validate_cfgs(target['cfgs']) return resources
def process(self, file_w_path, data=None): """ Processes the PinFile and any data (if a template) using Jinja2. Returns json of PinFile, topology, layout, and hooks. :param file_w_path: Full path to the provided file to process :param data: A JSON representation of data mapped to a Jinja2 template in file_w_path """ if not data: data = '{}' with open(file_w_path, 'r') as stream: file_data = stream.read() if data.startswith('@'): with open(data[1:], 'r') as strm: data = strm.read() try: file_data = self.render(file_data, data) return self.parse_json_yaml(file_data) except TypeError: error_txt = "Error attempting to parse PinFile data file." error_txt += "\nTemplate-data files require a prepended '@'" error_txt += " (eg. '@/path/to/template-data.yml')" error_txt += "\nPerhaps the path to the PinFile or" error_txt += " template-data is missing or the incorrect path?." raise ValidationError(error_txt) return self.load_pinfile(file_w_path)
def run_script(self, script): sp = None try: sp = subprocess.Popen(script, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except OSError as e: raise ValidationError("problem running {0} ({1})".format(script, e)) (stdout, stderr) = sp.communicate() if sp.returncode != 0: raise ValidationError("Script {0}" " had execution error".format(script)) return stdout
def _render_template(self, template, data): if data is None: return template if data.startswith('@'): with open(data[1:], 'r') as strm: data = strm.read() try: template = json.dumps(template) render = self.parser.render(template, data) except TypeError: error_txt = "Error attempting to parse PinFile data file." error_txt += "\nTemplate-data files require a prepended '@'" error_txt += " (eg. '@/path/to/template-data.yml')" error_txt += "\nPerhaps the path to the PinFile or" error_txt += " template-data is missing or the incorrect path?." raise ValidationError(error_txt) return json.JSONDecoder(object_pairs_hook=OrderedDict).decode(render)
def do_action(self, provision_data, action='up', run_id=None, tx_id=None): """ This function takes provision_data, and executes the given action for each target within the provision_data disctionary. :param provision_data: PinFile data as a dictionary, with target information :param action: Action taken (up, destroy, etc). (Default: up) :param run_id: Provided run_id to duplicate/destroy (Default: None) :param tx_id: Provided tx_id to duplicate/destroy (Default: None) .. .note:: The `run_id` value differs from the `rundb_id`, in that the `run_id` is an existing value in the database. The `rundb_id` value is created to store the new record. If the `run_id` is passed, it is used to collect an existing `uhash` value from the given `run_id`, which is in turn used to perform an idempotent reprovision, or destroy provisioned resources. """ ansible_console = False if self.ctx.cfgs.get('ansible'): ansible_console = (ast.literal_eval( self.get_cfg('ansible', 'console', default='False'))) if not ansible_console: ansible_console = bool(self.ctx.verbosity) results = {} return_code = 99 for target in provision_data.keys(): if not isinstance(provision_data[target], dict): raise LinchpinError("Target '{0}' does not" " exist.".format(target)) targets = [x.lower() for x in provision_data.keys()] if 'linchpin' in targets: raise LinchpinError("Target 'linchpin' is not allowed.") for target in provision_data.keys(): self.ctx.log_debug("Processing target: {0}".format(target)) results[target] = {} self.set_evar('target', target) rundb = self.setup_rundb() if tx_id: record = rundb.get_tx_record(tx_id) run_id = (record['targets'][0][target].keys()[0]) rundb_schema = json.loads( self.get_cfg(section='lp', key='rundb_schema')) rundb.schema = rundb_schema self.set_evar('rundb_schema', rundb_schema) start = time.time() st_uhash = int(start * 1000) uhash = None # generate a new rundb_id # (don't confuse it with an already existing run_id) rundb_id = rundb.init_table(target) orig_run_id = rundb_id uhash_length = self.get_cfg('lp', 'rundb_uhash_length') uhash_len = int(uhash_length) if not run_id: uh = hashlib.new( self.rundb_hash, ':'.join( [target, str(tx_id), str(rundb_id), str(st_uhash)])) uhash = uh.hexdigest()[:uhash_len] if action == 'destroy' or run_id: # look for the action='up' records to destroy data, orig_run_id = rundb.get_record(target, action='up', run_id=run_id) if data: uhash = data.get('uhash') self.ctx.log_debug("using data from" " run_id: {0}".format(run_id)) elif action not in ['up', 'destroy']: # it doesn't appear this code will will execute, # but if it does... raise LinchpinError("Attempting '{0}' action on" " target: '{1}' failed. Not an" " action.".format(action, target)) self.ctx.log_debug('rundb_id: {0}'.format(rundb_id)) self.ctx.log_debug('uhash: {0}'.format(uhash)) rundb.update_record(target, rundb_id, 'uhash', uhash) rundb.update_record(target, rundb_id, 'start', str(start)) rundb.update_record(target, rundb_id, 'action', action) self.set_evar('orig_run_id', orig_run_id) self.set_evar('rundb_id', rundb_id) self.set_evar('uhash', uhash) topology_data = provision_data[target].get('topology') # if validation fails the first time, convert topo from old -> new try: resources = self._validate_topology(topology_data) except (SchemaError, KeyError): # if topology fails, try converting from old to new style try: self._convert_topology(topology_data) resources = self._validate_topology(topology_data) except SchemaError: raise ValidationError("Topology '{0}' does not validate." "For more information run `linchpin" "validate`".format(topology_data)) self.set_evar('topo_data', topology_data) rundb.update_record( target, rundb_id, 'inputs', [{ 'topology_data': provision_data[target]['topology'] }]) if provision_data[target].get('layout', None): l_data = provision_data[target]['layout'] provision_data[target]['layout'] = self._convert_layout(l_data) self.set_evar('layout_data', provision_data[target]['layout']) rundb.update_record( target, rundb_id, 'inputs', [{ 'layout_data': provision_data[target]['layout'] }]) if provision_data[target].get('hooks', None): hooks_data = provision_data[target].get('hooks') self.set_evar('hooks_data', hooks_data) rundb.update_record( target, rundb_id, 'inputs', [{ 'hooks_data': provision_data[target]['hooks'] }]) if provision_data[target].get('cfgs', None): vars_data = provision_data[target].get('cfgs') self.set_evar('cfgs_data', vars_data) rundb.update_record(target, rundb_id, 'cfgs', [{ 'user': provision_data[target]['cfgs'] }]) # note : changing the state triggers the hooks self.hooks.rundb = (rundb, rundb_id) self.pb_hooks = self.get_cfg('hookstates', action) self.ctx.log_debug('calling: {0}{1}'.format('pre', action)) if 'pre' in self.pb_hooks: self.hook_state = '{0}{1}'.format('pre', action) # FIXME need to add rundb data for hooks results # invoke the appropriate action return_code, results[target]['task_results'] = ( self._invoke_playbooks(resources, action=action, console=ansible_console)) if not return_code: self.ctx.log_state("Action '{0}' on Target '{1}' is " "complete".format(action, target)) # FIXME Check the result[target] value here, and fail if applicable. # It's possible that a flag might allow more targets to run, then # return an error code at the end. # add post provision hook for inventory generation if 'inv' in self.pb_hooks: self.hook_state = 'postinv' if ('post' in self.pb_hooks) and (self.__meta__ == "API"): self.hook_state = '{0}{1}'.format('post', action) end = time.time() rundb.update_record(target, rundb_id, 'end', str(end)) rundb.update_record(target, rundb_id, 'rc', return_code) run_data = rundb.get_record(target, action=action, run_id=rundb_id) results[target]['rundb_data'] = {rundb_id: run_data[0]} # generate the linchpin_id and structure lp_schema = ('{"action": "", "targets": []}') rundb = self.setup_rundb() rundb.schema = json.loads(lp_schema) lp_id = rundb.init_table('linchpin') summary = {} for target, data in results.iteritems(): for k, v in data['rundb_data'].iteritems(): summary[target] = {k: {'rc': v['rc'], 'uhash': v['uhash']}} rundb.update_record('linchpin', lp_id, 'action', action) rundb.update_record('linchpin', lp_id, 'targets', [summary]) lp_data = { lp_id: { 'action': action, 'summary_data': summary, 'results_data': results } } return (return_code, lp_data)
def validate_pretty(self, target, name, old_schema=False): results = {} return_code = 0 for field in target.keys(): if field not in Validator.SECTIONS: raise ValidationError("Section '{0}' not a valid top-level " "PinFile section. valid sections " "are '{1}'".format( target, Validator.SECTIONS)) if 'topology' not in list(target.keys()): results['topology'] = "Each target must have a topology" err_prefix = "errors:\n" topo_data = target['topology'] try: self.validate_topology(topo_data) except (SchemaError, KeyError) as e: # if topology fails, try converting from old to new style try: self._convert_topology(topo_data) self.validate_topology(topo_data) except SchemaError as s: if old_schema: # add a tab to the beginning of each line in the # SchemaError and append to the existing error message error = self._format_error(err_prefix, s) else: if type(e) == KeyError: error = "\tfield res_defs['type'] is no longer "\ "supported. Please use 'role' instead" error = self._format_error(err_prefix, e) else: error = self._format_error(err_prefix, e) results['topology'] = error return_code += 1 else: results['topology'] = "valid under old schema" except TopologyError as t: error = self._format_error(err_prefix, t) results['topology'] = error return_code += 1 else: results['topology'] = "valid" if 'layout' in list(target.keys()): layout_data = target['layout'] try: self.validate_layout(layout_data) except SchemaError as e: error = self._format_error(err_prefix, e) results['layout'] = error + '\n' return_code += 1 else: results['layout'] = "valid" if 'cfgs' in list(target.keys()): cfgs_data = target['cfgs'] try: self.validate_cfgs(cfgs_data) except SchemaError as e: error = self._format_error(err_prefix, e) results['cfgs'] = error + '\n' return_code += 1 else: results['cfgs'] = "valid" return return_code, results
def do_action(self, provision_data, action='up', run_id=None, tx_id=None): """ This function takes provision_data, and executes the given action for each target within the provision_data disctionary. :param provision_data: PinFile data as a dictionary, with target information :param action: Action taken (up, destroy, etc). (Default: up) :param run_id: Provided run_id to duplicate/destroy (Default: None) :param tx_id: Provided tx_id to duplicate/destroy (Default: None) .. .note:: The `run_id` value differs from the `rundb_id`, in that the `run_id` is an existing value in the database. The `rundb_id` value is created to store the new record. If the `run_id` is passed, it is used to collect an existing `uhash` value from the given `run_id`, which is in turn used to perform an idempotent reprovision, or destroy provisioned resources. """ results = {} return_code = 99 # verify_targets_exist() targets = [x.lower() for x in list(provision_data.keys())] if 'linchpin' in targets: raise LinchpinError("Target 'linchpin' is not allowed.") for target in targets: if target == 'cfgs': continue if not isinstance(provision_data[target], dict): raise LinchpinError("Target '{0}' does not" " exist.".format(target)) self.ctx.log_debug("Processing target: {0}".format(target)) results[target] = {} self.set_evar('target', target) rundb_id = self.prepare_rundb(target, action, run_id, tx_id) try: validator = Validator(self.ctx, self.role_path, self.pb_ext) resources = validator.validate(provision_data[target]) except SchemaError: raise ValidationError("Target '{0}' does not validate. For" " more information run `linchpin " "validate`".format(target)) topology_data = provision_data[target].get('topology') self.set_evar('topo_data', topology_data) self.update_rundb(rundb_id, target, provision_data) # note : changing the state triggers the hooks return_code, results[target]['task_results'] = (self.run_target( target, resources, action, run_id) ) end = time.time() self.rundb.update_record(target, rundb_id, 'end', str(end)) self.rundb.update_record(target, rundb_id, 'rc', return_code) run_data = self.rundb.get_record(target, action=action, run_id=rundb_id) results[target]['rundb_data'] = {rundb_id: run_data[0]} # generate the linchpin_id and structure lp_data = self.write_results_to_rundb(results, action) return (return_code, lp_data)