def validate(self, target, old_schema=False):
"""
Attemptes to validate a single target in a PinFile
Returns the groups returned when the topology is validated
:param target: the target being validated
:param old_schema: whether or not to validate using the old schema
by default
"""
for field in target.keys():
if field not in Validator.SECTIONS:
raise ValidationError("Section '{0}' not a valid top-level "
"PinFile section. valid sections "
"are '{1}'".format(
target, Validator.SECTIONS))
if 'topology' not in list(target.keys()):
raise ValidationError("Each target must have a topology")
topo_data = target['topology']
try:
resources = self.validate_topology(topo_data)
except (SchemaError, KeyError):
# if topology fails, try converting from old to new style
self._convert_topology(topo_data)
resources = self.validate_topology(topo_data)
if 'layout' in list(target.keys()):
self.validate_layout(target['layout'])
if 'cfgs' in list(target.keys()):
self.validate_cfgs(target['cfgs'])
return resources
def process(self, file_w_path, data=None):
""" Processes the PinFile and any data (if a template)
using Jinja2. Returns json of PinFile, topology, layout,
and hooks.
:param file_w_path:
Full path to the provided file to process
:param data:
A JSON representation of data mapped to a Jinja2 template in
file_w_path
"""
if not data:
data = '{}'
with open(file_w_path, 'r') as stream:
file_data = stream.read()
if data.startswith('@'):
with open(data[1:], 'r') as strm:
data = strm.read()
try:
file_data = self.render(file_data, data)
return self.parse_json_yaml(file_data)
except TypeError:
error_txt = "Error attempting to parse PinFile data file."
error_txt += "\nTemplate-data files require a prepended '@'"
error_txt += " (eg. '@/path/to/template-data.yml')"
error_txt += "\nPerhaps the path to the PinFile or"
error_txt += " template-data is missing or the incorrect path?."
raise ValidationError(error_txt)
return self.load_pinfile(file_w_path)
def run_script(self, script):
sp = None
try:
sp = subprocess.Popen(script,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError as e:
raise ValidationError("problem running {0} ({1})".format(script, e))
(stdout, stderr) = sp.communicate()
if sp.returncode != 0:
raise ValidationError("Script {0}"
" had execution error".format(script))
return stdout
def _render_template(self, template, data):
if data is None:
return template
if data.startswith('@'):
with open(data[1:], 'r') as strm:
data = strm.read()
try:
template = json.dumps(template)
render = self.parser.render(template, data)
except TypeError:
error_txt = "Error attempting to parse PinFile data file."
error_txt += "\nTemplate-data files require a prepended '@'"
error_txt += " (eg. '@/path/to/template-data.yml')"
error_txt += "\nPerhaps the path to the PinFile or"
error_txt += " template-data is missing or the incorrect path?."
raise ValidationError(error_txt)
return json.JSONDecoder(object_pairs_hook=OrderedDict).decode(render)
def do_action(self, provision_data, action='up', run_id=None, tx_id=None):
"""
This function takes provision_data, and executes the given
action for each target within the provision_data disctionary.
:param provision_data: PinFile data as a dictionary, with
target information
:param action: Action taken (up, destroy, etc). (Default: up)
:param run_id: Provided run_id to duplicate/destroy (Default: None)
:param tx_id: Provided tx_id to duplicate/destroy (Default: None)
.. .note:: The `run_id` value differs from the `rundb_id`, in that
the `run_id` is an existing value in the database.
The `rundb_id` value is created to store the new record.
If the `run_id` is passed, it is used to collect an existing
`uhash` value from the given `run_id`, which is in turn used
to perform an idempotent reprovision, or destroy provisioned
resources.
"""
ansible_console = False
if self.ctx.cfgs.get('ansible'):
ansible_console = (ast.literal_eval(
self.get_cfg('ansible', 'console', default='False')))
if not ansible_console:
ansible_console = bool(self.ctx.verbosity)
results = {}
return_code = 99
for target in provision_data.keys():
if not isinstance(provision_data[target], dict):
raise LinchpinError("Target '{0}' does not"
" exist.".format(target))
targets = [x.lower() for x in provision_data.keys()]
if 'linchpin' in targets:
raise LinchpinError("Target 'linchpin' is not allowed.")
for target in provision_data.keys():
self.ctx.log_debug("Processing target: {0}".format(target))
results[target] = {}
self.set_evar('target', target)
rundb = self.setup_rundb()
if tx_id:
record = rundb.get_tx_record(tx_id)
run_id = (record['targets'][0][target].keys()[0])
rundb_schema = json.loads(
self.get_cfg(section='lp', key='rundb_schema'))
rundb.schema = rundb_schema
self.set_evar('rundb_schema', rundb_schema)
start = time.time()
st_uhash = int(start * 1000)
uhash = None
# generate a new rundb_id
# (don't confuse it with an already existing run_id)
rundb_id = rundb.init_table(target)
orig_run_id = rundb_id
uhash_length = self.get_cfg('lp', 'rundb_uhash_length')
uhash_len = int(uhash_length)
if not run_id:
uh = hashlib.new(
self.rundb_hash, ':'.join(
[target,
str(tx_id),
str(rundb_id),
str(st_uhash)]))
uhash = uh.hexdigest()[:uhash_len]
if action == 'destroy' or run_id:
# look for the action='up' records to destroy
data, orig_run_id = rundb.get_record(target,
action='up',
run_id=run_id)
if data:
uhash = data.get('uhash')
self.ctx.log_debug("using data from"
" run_id: {0}".format(run_id))
elif action not in ['up', 'destroy']:
# it doesn't appear this code will will execute,
# but if it does...
raise LinchpinError("Attempting '{0}' action on"
" target: '{1}' failed. Not an"
" action.".format(action, target))
self.ctx.log_debug('rundb_id: {0}'.format(rundb_id))
self.ctx.log_debug('uhash: {0}'.format(uhash))
rundb.update_record(target, rundb_id, 'uhash', uhash)
rundb.update_record(target, rundb_id, 'start', str(start))
rundb.update_record(target, rundb_id, 'action', action)
self.set_evar('orig_run_id', orig_run_id)
self.set_evar('rundb_id', rundb_id)
self.set_evar('uhash', uhash)
topology_data = provision_data[target].get('topology')
# if validation fails the first time, convert topo from old -> new
try:
resources = self._validate_topology(topology_data)
except (SchemaError, KeyError):
# if topology fails, try converting from old to new style
try:
self._convert_topology(topology_data)
resources = self._validate_topology(topology_data)
except SchemaError:
raise ValidationError("Topology '{0}' does not validate."
"For more information run `linchpin"
"validate`".format(topology_data))
self.set_evar('topo_data', topology_data)
rundb.update_record(
target, rundb_id, 'inputs',
[{
'topology_data': provision_data[target]['topology']
}])
if provision_data[target].get('layout', None):
l_data = provision_data[target]['layout']
provision_data[target]['layout'] = self._convert_layout(l_data)
self.set_evar('layout_data', provision_data[target]['layout'])
rundb.update_record(
target, rundb_id, 'inputs',
[{
'layout_data': provision_data[target]['layout']
}])
if provision_data[target].get('hooks', None):
hooks_data = provision_data[target].get('hooks')
self.set_evar('hooks_data', hooks_data)
rundb.update_record(
target, rundb_id, 'inputs',
[{
'hooks_data': provision_data[target]['hooks']
}])
if provision_data[target].get('cfgs', None):
vars_data = provision_data[target].get('cfgs')
self.set_evar('cfgs_data', vars_data)
rundb.update_record(target, rundb_id, 'cfgs',
[{
'user': provision_data[target]['cfgs']
}])
# note : changing the state triggers the hooks
self.hooks.rundb = (rundb, rundb_id)
self.pb_hooks = self.get_cfg('hookstates', action)
self.ctx.log_debug('calling: {0}{1}'.format('pre', action))
if 'pre' in self.pb_hooks:
self.hook_state = '{0}{1}'.format('pre', action)
# FIXME need to add rundb data for hooks results
# invoke the appropriate action
return_code, results[target]['task_results'] = (
self._invoke_playbooks(resources,
action=action,
console=ansible_console))
if not return_code:
self.ctx.log_state("Action '{0}' on Target '{1}' is "
"complete".format(action, target))
# FIXME Check the result[target] value here, and fail if applicable.
# It's possible that a flag might allow more targets to run, then
# return an error code at the end.
# add post provision hook for inventory generation
if 'inv' in self.pb_hooks:
self.hook_state = 'postinv'
if ('post' in self.pb_hooks) and (self.__meta__ == "API"):
self.hook_state = '{0}{1}'.format('post', action)
end = time.time()
rundb.update_record(target, rundb_id, 'end', str(end))
rundb.update_record(target, rundb_id, 'rc', return_code)
run_data = rundb.get_record(target, action=action, run_id=rundb_id)
results[target]['rundb_data'] = {rundb_id: run_data[0]}
# generate the linchpin_id and structure
lp_schema = ('{"action": "", "targets": []}')
rundb = self.setup_rundb()
rundb.schema = json.loads(lp_schema)
lp_id = rundb.init_table('linchpin')
summary = {}
for target, data in results.iteritems():
for k, v in data['rundb_data'].iteritems():
summary[target] = {k: {'rc': v['rc'], 'uhash': v['uhash']}}
rundb.update_record('linchpin', lp_id, 'action', action)
rundb.update_record('linchpin', lp_id, 'targets', [summary])
lp_data = {
lp_id: {
'action': action,
'summary_data': summary,
'results_data': results
}
}
return (return_code, lp_data)
def validate_pretty(self, target, name, old_schema=False):
results = {}
return_code = 0
for field in target.keys():
if field not in Validator.SECTIONS:
raise ValidationError("Section '{0}' not a valid top-level "
"PinFile section. valid sections "
"are '{1}'".format(
target, Validator.SECTIONS))
if 'topology' not in list(target.keys()):
results['topology'] = "Each target must have a topology"
err_prefix = "errors:\n"
topo_data = target['topology']
try:
self.validate_topology(topo_data)
except (SchemaError, KeyError) as e:
# if topology fails, try converting from old to new style
try:
self._convert_topology(topo_data)
self.validate_topology(topo_data)
except SchemaError as s:
if old_schema:
# add a tab to the beginning of each line in the
# SchemaError and append to the existing error message
error = self._format_error(err_prefix, s)
else:
if type(e) == KeyError:
error = "\tfield res_defs['type'] is no longer "\
"supported. Please use 'role' instead"
error = self._format_error(err_prefix, e)
else:
error = self._format_error(err_prefix, e)
results['topology'] = error
return_code += 1
else:
results['topology'] = "valid under old schema"
except TopologyError as t:
error = self._format_error(err_prefix, t)
results['topology'] = error
return_code += 1
else:
results['topology'] = "valid"
if 'layout' in list(target.keys()):
layout_data = target['layout']
try:
self.validate_layout(layout_data)
except SchemaError as e:
error = self._format_error(err_prefix, e)
results['layout'] = error + '\n'
return_code += 1
else:
results['layout'] = "valid"
if 'cfgs' in list(target.keys()):
cfgs_data = target['cfgs']
try:
self.validate_cfgs(cfgs_data)
except SchemaError as e:
error = self._format_error(err_prefix, e)
results['cfgs'] = error + '\n'
return_code += 1
else:
results['cfgs'] = "valid"
return return_code, results
def do_action(self, provision_data, action='up', run_id=None, tx_id=None):
"""
This function takes provision_data, and executes the given
action for each target within the provision_data disctionary.
:param provision_data: PinFile data as a dictionary, with
target information
:param action: Action taken (up, destroy, etc). (Default: up)
:param run_id: Provided run_id to duplicate/destroy (Default: None)
:param tx_id: Provided tx_id to duplicate/destroy (Default: None)
.. .note:: The `run_id` value differs from the `rundb_id`, in that
the `run_id` is an existing value in the database.
The `rundb_id` value is created to store the new record.
If the `run_id` is passed, it is used to collect an existing
`uhash` value from the given `run_id`, which is in turn used
to perform an idempotent reprovision, or destroy provisioned
resources.
"""
results = {}
return_code = 99
# verify_targets_exist()
targets = [x.lower() for x in list(provision_data.keys())]
if 'linchpin' in targets:
raise LinchpinError("Target 'linchpin' is not allowed.")
for target in targets:
if target == 'cfgs':
continue
if not isinstance(provision_data[target], dict):
raise LinchpinError("Target '{0}' does not"
" exist.".format(target))
self.ctx.log_debug("Processing target: {0}".format(target))
results[target] = {}
self.set_evar('target', target)
rundb_id = self.prepare_rundb(target, action, run_id, tx_id)
try:
validator = Validator(self.ctx, self.role_path, self.pb_ext)
resources = validator.validate(provision_data[target])
except SchemaError:
raise ValidationError("Target '{0}' does not validate. For"
" more information run `linchpin "
"validate`".format(target))
topology_data = provision_data[target].get('topology')
self.set_evar('topo_data', topology_data)
self.update_rundb(rundb_id, target, provision_data)
# note : changing the state triggers the hooks
return_code, results[target]['task_results'] = (self.run_target(
target,
resources,
action,
run_id)
)
end = time.time()
self.rundb.update_record(target, rundb_id, 'end', str(end))
self.rundb.update_record(target, rundb_id, 'rc', return_code)
run_data = self.rundb.get_record(target,
action=action,
run_id=rundb_id)
results[target]['rundb_data'] = {rundb_id: run_data[0]}
# generate the linchpin_id and structure
lp_data = self.write_results_to_rundb(results, action)
return (return_code, lp_data)
