def __init__(self, ctx, fetch_type, src, dest, cache_dir, root='', root_ws='', ref=None): super(FetchHttp, self).__init__(ctx, fetch_type, dest, root=root, root_ws=root_ws, ref=ref) self.cache_dir = os.path.join(cache_dir, "http") if not os.path.exists(self.cache_dir): os.makedirs(self.cache_dir) try: r = requests.get(src) r.raise_for_status() except requests.exceptions.HTTPError: raise LinchpinError("An HTTP error occurred") except requests.exceptions.RequestException: raise LinchpinError("Could not connect to given URL") if r.status_code != 200: raise LinchpinError("The entered url is invalid") self.src = src.rstrip('/')
def do_validation(self, provision_data, old_schema=False): """ This function takes provision_data, and attempts to validate the topologies for that data :param provision_data: PinFile data as a dictionary, with target information """ results = {} return_code = 0 for target in provision_data.keys(): if not isinstance(provision_data[target], dict): raise LinchpinError("Target '{0}' does not" " exist.".format(target)) targets = [x.lower() for x in provision_data.keys()] if 'linchpin' in targets: raise LinchpinError("Target 'linchpin' is not allowed.") for target in provision_data.keys(): self.ctx.log_debug("Processing target: {0}".format(target)) results[target] = {} self.set_evar('target', target) validator = Validator(self.ctx, self.pb_path, self.pb_ext) target_dict = provision_data[target] ret, results[target] = validator.validate_pretty( target_dict, target, old_schema) return_code += ret return return_code, results
def _invoke_playbooks(self, resources={}, action='up', console=True, providers=[]): """ Uses the Ansible API code to invoke the specified linchpin playbook :param resources: dict of resources to provision :param action: Which ansible action to run (default: 'up') :param console: Whether to display the ansible console (default: True) """ return_code = 0 results = [] self.set_evar('_action', action) self.set_evar('state', 'present') if action == 'setup' or action == 'ask_sudo_setup': self.set_evar('setup_providers', providers) return_code, res = self._find_n_run_pb(action, "localhost", console=console) if res: results.append(res) if not len(results): results = None return (return_code, results) if action == 'destroy': self.set_evar('state', 'absent') inventory_src = '{0}/localhost'.format(self.workspace) for resource in resources: self.set_evar('resources', resource) playbook = resource.get('resource_group_type') return_code, res = self._find_n_run_pb(playbook, inventory_src, console=console) if action == "up" and (return_code > 0 and not isinstance(return_code, str)): if self.ctx.verbosity > 0: raise LinchpinError("Unsuccessful provision of resource") else: res_grp_name = resource['resource_group_name'] msg = res['failed'][0]._result['msg'] task = res['failed'][0].task_name raise LinchpinError("Unable to provision resource group " "'{0}' due to '{1}' at task " " '{2}'".format(res_grp_name, msg, task)) sys.exit(return_code) if res: results.append(res) if not len(results): results = None return (return_code, results)
def __init__(self, ctx, fetch_type, src, dest, cache_dir, root): super(FetchLocal, self).__init__(ctx, fetch_type, dest, root) self.cache_dir = os.path.join(cache_dir, "local") if not os.path.exists(self.cache_dir): os.mkdir(self.cache_dir) src_parse = urlparse.urlparse(src) self.src = os.path.abspath( os.path.join(src_parse.netloc, src_parse.path)) if not os.path.exists(self.src): raise LinchpinError('{0} is not a valid path'.format(src)) if os.path.samefile(self.src, self.dest): raise LinchpinError("Provide two different locations")
def call_wget(self, fetch_dir=None): src_w_root = '{0}/{1}'.format(self.src, self.root) tempdir = None # globs to reject rej = '*html*' if not fetch_dir: fetch_dir = tempfile.mkdtemp(prefix="http_", dir=self.cache_dir) wget_args = ['wget', '-r', '-np', '-nH', '-q', '--reject', rej, src_w_root, '-P', fetch_dir] retval = subprocess.call(wget_args) if retval != 0: try: os.rmdir(tempdir) except OSError: pass raise LinchpinError('Unable to fetch files with the following' ' command:\n{0}'.format(" ".join(wget_args))) return fetch_dir
def call_wget(self, src, fetch_dir=None): list_args = src.split('/') list_args = list_args[3:] tempdir = None if fetch_dir is None: tempdir = tempfile.mkdtemp(prefix="http_", dir=self.cache_dir) wget_args = ['wget', '-r', '-np', '-nH', '-q', '--reject', 'html', '--cut-dirs={0}'.format(len(list_args)), src, '-P', tempdir] else: tempdir = fetch_dir wget_args = ['wget', '-r', '-np', '-N', '-nH', '-q', '--reject', 'html', '--cut-dirs={0}'.format(len(list_args)), src, '-P', tempdir] retval = subprocess.call(wget_args) if retval != 0: try: os.rmdir(tempdir) except OSError: pass raise LinchpinError('Unable to fetch files with the following' ' command:\n{0}'.format(" ".join(wget_args))) return tempdir
def find_include(self, filename, ftype='topology'): """ Find the included file to be acted upon. :param filename: name of file from to be loaded :param ftype: the file type to locate: topology, layout (default: topology) """ folder = self.get_evar('topologies_folder', 'topologies') if ftype == 'layout': folder = self.get_evar('layouts_folder', 'layouts') elif ftype == 'hooks': folder = self.get_evar('hooks_folder', 'hooks') path = os.path.realpath('{0}/{1}'.format(self.workspace, folder)) files = os.listdir(path) if filename in files: return os.path.realpath('{0}/{1}'.format(path, filename)) raise LinchpinError('{0} not found in' ' workspace'.format(filename))
def call_clone(self, fetch_dir=None): ref = None src = self.src if self.ref: ref = self.ref src = '{0}@{1}'.format(self.src, ref) if fetch_dir and os.path.exists(fetch_dir): cmd = ['git', 'pull', '--quiet'] retval = subprocess.call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=fetch_dir) else: if not fetch_dir: fetch_dir = tempfile.mkdtemp(prefix="git_", dir=self.cache_dir) cmd = ['git', 'clone', '--quiet', self.src] if ref: cmd.extend(['-b', ref]) cmd.append(fetch_dir) retval = subprocess.call(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if retval != 0: raise LinchpinError("Unable to clone {0}".format(src)) return fetch_dir
def copy_dir(self, src, dest): for root, dirs, files in os.walk(src): files = [f for f in files if not f[0] == '.'] dirs[:] = [d for d in dirs if not d[0] == '.'] if not os.path.isdir(root): os.makedirs(root) for file in files: rel_path = root.replace(src, '').lstrip(os.sep) dest_path = os.path.join(dest, rel_path) if not os.path.isdir(dest_path): try: os.makedirs(dest_path) except OSError as e: if e.errno == 17: raise LinchpinError( 'File {0} already exists' ' in destination directory'.format(e.filename)) s = os.path.join(root, file) d = os.path.join(dest_path, file) if (not os.path.exists(d)) or ( os.stat(d).st_mtime - os.stat(dest).st_mtime > 1): try: if os.path.islink(s) and os.path.exists( os.readlink(s)): linkto = os.readlink(s) os.symlink(linkto, d) else: shutil.copy2(s, d) except (IOError, OSError): pass
def get_driver(provider): if provider not in filter_classes: raise LinchpinError("Key {0} not found in" " inventory provider dict".format(provider)) return filter_classes[provider]
def validate_topology_highlevel(self, topo_data): """ validate the higher-level components of the topology These are not specific to the provider and must be validated separately from the items within each resource group :param topo_data topology data from the pinfile """ role_path = self._find_role_path("common") try: sp = "{0}/files/topo-schema.json".format(role_path) schema = json.load(open(sp)) except Exception as e: raise LinchpinError("Error with schema: '{0}'" " {1}".format(sp, e)) document = {'topology': topo_data} v = AnyofValidator(schema, error_handler=ValidationErrorHandler) if not v.validate(document): try: err = self._gen_error_msg("", "", v.errors) raise TopologyError(err) except NotImplementedError: # we shouldn't have this issue using cererus >= 1.2, but # this is here just in case an older version has to be used self.ctx.log_state("There was an error validating your schema,\ but we can't seem to format it for you") self.ctx.log_state("Here's the raw error data in case you want\ to go through it by hand:") self.ctx.log_state(v._errors)
def _parse_config(self, path): """ Parse configs into the self.cfgs dict from provided path. :param path: A path to a config to parse """ try: config = ConfigParser.ConfigParser() f = open(path) config.read_file(f) f.close() for section in config.sections(): if not self.cfgs.get(section): self.cfgs[section] = {} for k in config.options(section): if section == 'evars': try: self.cfgs[section][k] = (config.getboolean( section, k)) except ValueError: self.cfgs[section][k] = config.get(section, k) else: try: self.cfgs[section][k] = config.get(section, k) except ConfigParser.InterpolationMissingOptionError: value = config.get(section, k, raw=True) self.cfgs[section][k] = value.replace('%%', '%') except ConfigParser.InterpolationSyntaxError as e: raise LinchpinError('Unable to parse configuration file properly:' ' {0}'.format(e))
def _get_pinfile_path(self, exists=True): """ This function finds the self.pinfile. If the file is a full path, it is expanded and used. If not found, the lp.default_pinfile configuration value is used for the pinfile, the workspace is prepended and returned. :param exists: Whether the pinfile is supposed to already exist (default: True) """ if not self.pinfile: self.pinfile = self.get_cfg('lp', 'default_pinfile', default='PinFile') pf_w_path = os.path.realpath(os.path.expanduser(self.pinfile)) if self.workspace: pf_w_path = '{0}/{1}'.format(self.workspace, self.pinfile) # Ensure a PinFile path will exist if not os.path.exists(pf_w_path) and exists: pf_w_path = '{0}/{1}'.format(self.workspace, self.pinfile) # If the PinFile doesn't exist, raise an error if not os.path.exists(pf_w_path) and exists: raise LinchpinError('{0} not found. Please check that it' ' exists and try again'.format(pf_w_path)) return pf_w_path
def _find_playbook_path(self, playbook): p = '{0}/{1}'.format(self.pb_path, playbook) if os.path.exists(os.path.expanduser(p)): return self.pb_path raise LinchpinError("playbook '{0}' not found in" " path: {1}".format(playbook, self.pb_path))
def write_config_file(self, path): try: with open(path, 'a+') as f: self.parser.write(f) except Exception as e: raise LinchpinError('Unable to write configuration file:' ' {0}'.format(e))
def lp_fetch(self, src, root=None, fetch_type='workspace'): if root is not None: root = list(filter(None, root.split(','))) dest = self.workspace if not os.path.exists(dest): raise LinchpinError(dest + " does not exist") fetch_aliases = { "topologies": self.get_evar("topologies_folder"), "layouts": self.get_evar("layouts_folder"), "resources": self.get_evar("resources_folder"), "hooks": self.get_evar("hooks_folder"), "workspace": "workspace" } fetch_dir = fetch_aliases.get(fetch_type, "workspace") cache_path = os.path.abspath(os.path.join(os.path.expanduser('~'), '.cache/linchpin')) if not os.path.exists(cache_path): os.makedirs(cache_path) protocol_regex = OrderedDict([ ('((git|ssh|http(s)?)|(git@[\w\.]+))' '(:(//)?)([\w\.@\:/\-~]+)(\.git)(/)?', 'FetchGit'), ('^(http|https)://', 'FetchHttp'), ('^(file)://', 'FetchLocal') ]) fetch_protocol = None for regex, obj in protocol_regex.items(): if re.match(regex, src): fetch_protocol = obj break if fetch_protocol is None: raise LinchpinError("The protocol speficied is not supported") fetch_class = FETCH_CLASS[fetch_protocol](self.ctx, fetch_dir, src, dest, cache_path, root) fetch_class.fetch_files() fetch_class.copy_files()
def yaml2json(pf): """ parses yaml file into json object """ with open(pf, 'r') as stream: try: pf = yaml.load(stream) return pf except yaml.YAMLError as exc: raise LinchpinError(exc)
def copy_dir(self, src, dest): for root, dirs, files in os.walk(src): files = [f for f in files if not f[0] == '.'] dirs[:] = [d for d in dirs if not d[0] == '.'] if not os.path.isdir(root): os.makedirs(root) for f in files: rel_path = root.replace(src, '').lstrip(os.sep) dest_path = os.path.join(dest, rel_path) if not os.path.isdir(dest_path): try: os.makedirs(dest_path) except OSError as e: if e.errno == 17: raise LinchpinError('File {0} already exists' ' in destination directory' .format(e.filename)) s_file = os.path.join(root, f) d_file = os.path.join(dest_path, f) # fetch.always_update_workspace flag determines whether or # not to update. can be overwritten on the cli with --nocache. cache_ws = (ast.literal_eval( self.ctx.get_cfg('fetch', 'cache_ws', default='True'))) cp_files = False if not cache_ws: cp_files = True else: if not os.path.exists(d_file): cp_files = True else: cache_days = int(self.ctx.get_cfg('fetch', 'cache_days', default=1)) s_file_mtime = int(os.stat(s_file).st_mtime) d_file_mtime = int(os.stat(d_file).st_mtime) if (s_file_mtime - d_file_mtime) >= cache_days: cp_files = True if cp_files: try: if (os.path.islink(s_file) and os.path.exists(os.readlink(s_file))): s_file = os.readlink(s_file) shutil.copy2(s_file, d_file) except (IOError, OSError) as e: self.ctx.log_state(e)
def set_flag_no_hooks(self, flag): """ set_flag_no_hooks sets no_hooks flag param: flag: boolean returns boolean """ if isinstance(flag, bool): return self.context.set_cfg("hookflags", "no_hooks", flag) raise LinchpinError("Incorrect datatype please use boolean")
def _invoke_playbooks(self, resources={}, action='up', console=True, providers=[]): """ Uses the Ansible API code to invoke the specified linchpin playbook :param resources: dict of resources to provision :param action: Which ansible action to run (default: 'up') :param console: Whether to display the ansible console (default: True) """ return_code = 0 results = [] self.set_evar('_action', action) self.set_evar('state', 'present') if action == 'setup' or action == 'ask_sudo_setup': self.set_evar('setup_providers', providers) return_code, res = self._find_n_run_pb(action, "localhost", console=console) if res: results.append(res) if not len(results): results = None return (return_code, results) if action == 'destroy': self.set_evar('state', 'absent') inventory_src = '{0}/localhost'.format(self.workspace) for resource in resources: self.set_evar('resources', resource) playbook = resource.get('resource_group_type') return_code, res = self._find_n_run_pb(playbook, inventory_src, console=console) if action == "up" and return_code > 0: raise LinchpinError("Unsuccessful provision of resource " "System return: {0}".format(return_code)) sys.exit(return_code) if res: results.append(res) if not len(results): results = None return (return_code, results)
def set_flag_ignore_failed_hooks(self, flag): """ set_flag_ignore_failed_hooks sets current ignore_failed_hooks flag value param: flag: boolean """ if isinstance(flag, bool): return self.context.set_cfg("hookflags", "ignore_failed_hooks", flag) raise LinchpinError("Incorrect datatype please use boolean")
def set_vault_encryption(self, vault_enc): """ set_vault_encryption sets vault_encryption flag if credentials are encrypted in vault current credentials path param: vault_enc: boolean returns boolean """ if isinstance(vault_enc, bool): return self.context.set_evar("vault_encryption", vault_enc) raise LinchpinError("Incorrect datatype please use boolean")
def call_clone(self, fetch_dir=None): if fetch_dir: retval = subprocess.call( ['git', '-C', fetch_dir, 'pull', '--quiet']) tempdir = fetch_dir else: tempdir = tempfile.mkdtemp(prefix="git_", dir=self.cache_dir) retval = subprocess.call( ['git', 'clone', '--quiet', self.src, tempdir]) if retval != 0: raise LinchpinError("Unable to clone {0}".format(self.src)) return tempdir
def set_credentials_path(self, creds_path): """ set_credentials_path function set credentials path :param creds_path: path to credential directory returns True/False """ if os.path.isdir(creds_path): return self.context.set_evar("default_credentials_path", creds_path) raise LinchpinError("Incorrect file path, path should be a directory")
def _convert_topology(self, topology): """ For backward compatiblity, convert the old topology format into the new format. Should be pretty straightforward and simple. ;param topology: topology dictionary """ try: res_grps = topology.get('resource_groups') if res_grps: for res_grp in res_grps: if 'res_group_type' in res_grp.keys(): res_grp['resource_group_type'] = ( res_grp.pop('res_group_type')) if 'res_defs' in res_grp.keys(): res_grp['resource_definitions'] = ( res_grp.pop('res_defs')) res_defs = res_grp.get('resource_definitions') if not res_defs: # this means it's either a beaker or openshift topology res_grp_type = res_grp.get('resource_group_type') res_group = self._fix_broken_topologies( res_grp, res_grp_type) res_defs = res_group.get('resource_definitions') res_grp['resource_definitions'] = res_defs if res_defs: for res_def in res_defs: if 'res_name' in res_def.keys(): res_def['name'] = res_def.pop('res_name') if 'type' in res_def.keys(): res_def['role'] = res_def.pop('type') if 'res_type' in res_def.keys(): res_def['role'] = res_def.pop('res_type') if 'count' in res_def.keys(): res_def['count'] = int(res_def.pop('count')) else: raise TopologyError("'resource_definitions' do not" " validate in topology" " ({0})".format(topology)) else: raise TopologyError("'resource_groups' do not validate" " in topology ({0})".format(topology)) except Exception: raise LinchpinError("Unknown error converting schema. Check" " template data")
def parse_json_yaml(self, data): """ parses yaml file into json object """ d = None try: data = yaml.load(data, Loader=yamlordereddictloader.Loader) except Exception as e: raise LinchpinError('YAML parsing error: {}'.format(e)) if isinstance(data, dict): return data return d
def transfer_section(self, section): dest_dir = os.path.join(self.dest, section) dir_exists = True if section not in os.listdir(self.dest): dir_exists = False os.makedirs(dest_dir) src_dir = os.path.join('{0}/{1}'.format(self.td, self.root), section) if not os.path.exists(src_dir): if not dir_exists: shutil.rmtree(dest_dir) raise LinchpinError('The {0} directory does not exist in ' '{1}'.format(self.fetch_type, self.src)) self.copy_dir(src_dir, dest_dir)
def _get_role(self, role): for path in self.role_path: p = '{0}/{1}'.format(path, role) if os.path.exists(os.path.expanduser(p)): return # if the role is not in role_path, ansible-galaxy may be able to install # it (if it has not already). galaxy_runner() will return True if the # role was successfully installed or if the role was previously # installed. It will return false otherwise if not galaxy_runner.install(role): raise LinchpinError("role '{0}' not found in path: {1}\n. It also" " could not be installed via Ansible" " Galaxy".format(role, self.role_path))
def _find_playbook_path(self, playbook): """ returns the full path to a given playbook :params playbook: name of the playbook """ for path in self.pb_path: p = '{0}/{1}{2}'.format(path, playbook, self.pb_ext) if os.path.exists(os.path.expanduser(p)): return path raise LinchpinError("playbook '{0}' not found in" " path: {1}".format(playbook, self.pb_path))
def _validate_topology(self, topology): """ Validate the provided topology against the schema ;param topology: topology dictionary """ res_grps = topology.get('resource_groups') resources = [] for group in res_grps: res_grp_type = (group.get('resource_group_type') or group.get('res_group_type')) pb_path = self._find_playbook_path(res_grp_type) try: sp = "{0}/roles/{1}/files/schema.json".format( pb_path, res_grp_type) schema = json.load(open(sp)) except Exception as e: raise LinchpinError("Error with schema: '{0}'" " {1}".format(sp, e)) res_defs = group.get('resource_definitions') # preload this so it will validate against the schema document = {'res_defs': res_defs} v = AnyofValidator(schema, error_handler=ValidationErrorHandler) if not v.validate(document): try: err = self._gen_error_msg("", "", v.errors) raise SchemaError(err) except NotImplementedError as e: # we shouldn't have this issue using cerberus >= 1.2, but # this is here just in case an older version has to be used print("There was an error validating your schema, but we\ can't seem to format it for you") print("Here's the raw error data in case you want to go\ through it by hand:") print(v._errors) resources.append(group) return resources