def get_ssh_client(hostname, ssh_hostname): """Tries to create ssh client Create ssh client based on the username and ssh key """ if not CREDS.SSH_KEYFILE: logger.errorout("ssh_keyfile not set", module=COMMAND_MODULE_CUSTOM) retries = 0 while retries < MAX_SSH_RETRIES: try: ssh = paramiko.SSHClient() ssh.load_system_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname=ssh_hostname, username=CREDS.SSH_USER, port=CREDS.SSH_PORT, pkey=CREDS.PK, timeout=CONNECTION_TIMEOUT) return ssh except paramiko.BadAuthenticationType: logger.error("BadAuthenticationType", hostname=hostname, module=COMMAND_MODULE_CUSTOM) return except paramiko.AuthenticationException: logger.error("Authentication failed", hostname=hostname, module=COMMAND_MODULE_CUSTOM) return except paramiko.BadHostKeyException: logger.error("BadHostKeyException", fix="Edit known_hosts file to remove the entry", hostname=hostname, module=COMMAND_MODULE_CUSTOM) return except paramiko.SSHException: logger.error("SSHException", hostname=hostname, module=COMMAND_MODULE_CUSTOM) return except Exception as e: if retries == 0: logger.error("Problems connecting to host", hostname=hostname, module=COMMAND_MODULE_CUSTOM, error=e.message) retries += 1 time.sleep(1) logger.error("Can not connect to host", hostname=hostname, module=COMMAND_MODULE_CUSTOM) return None
def from_params(self, _app, _app_clean, _method, _commit_id, _hostname, _is_firstrun=False): """populate application from params""" self._deployment_method = self.__deployment_mng.get_deployment_method( _method) if not self._deployment_method: logger.errorout("Deployment method for app invalid", app=_app, method=_method) self.app = _app self.app_clean = _app_clean self.source_hostname = _hostname self.method_info = self._deployment_method.data.copy() self._commit_id = None if _commit_id == 'N/A' or len( _commit_id) == 0 else _commit_id self.is_firstrun = _is_firstrun self.track = self.__repo_mng.track self.status = consts.META_APP_UNCHANGED self.updated = False self.app_creation_datetime = helpers.get_utc() self.content_type = helpers.get_update_str(False) return self
def template_directory(app_path, templating_values): """Template files Walks through all the files a directory and templates any jinja2 values found. """ if not check_path(app_path): logger.errorout("Can not copy location that does not exist", path=app_path) tvalues = merge_templates(templating_values) for path, _dir, files in os.walk(app_path): # sort files so logs read better and easier to get status files.sort() j2_env = Environment(autoescape=True, loader=FileSystemLoader(path)) for filename in files: # Should not template version file since it may have # regex commands that can break templating. if filename.startswith(consts.VERSIONS_FILENAME): continue file_path = os.path.join(path, filename) try: file_content = j2_env.get_template(filename).render(tvalues) with open(file_path, 'w') as f: f.write(file_content) except Exception as e: logger.errorout('Error templating file', file=file_path, error=e.message)
def pull_repo(self, force=False): """Clone repo to specified dir. Delete repo if it currently exist unless reuse. """ try: helpers.create_path(self.paths['absolute_path'], True) if force: self.delete_repo() if not os.path.exists(self.paths['repo_path']): logger.info("Starting Repo Cloning", track=self.track) output, rc = helpers.run( "git clone -b %s %s" % (self.branch, self.url), self.paths['absolute_path'], self.dryrun) if rc > 0: self.delete_repo() logger.error("Pulling_repo", error=output, path=self.paths['repo_path']) return -1 return 1 else: return 0 except Exception as e: logger.errorout("Pulling_repo", err_msg=e.message, error="Error pulling repo", path=self.paths['repo_path'])
def get_commit_log(self): """Get the current commit log """ try: log_object = {} for key, value in COMMIT_KEYS.items(): stdout, _rc = helpers.run( ['git', 'log', '-1', '--pretty=\'%s\'' % value], self.paths['repo_path'], self.dryrun) output = "XXXXX" if self.dryrun else helpers.filter_content( stdout) if key in consts.RENAME_COMMIT_LOG_KEYS: key = consts.RENAME_COMMIT_LOG_KEYS[key] log_object[key] = output log_object['project'] = self.project log_object['reponame'] = self.reponame return log_object except Exception as e: logger.errorout("get_commit_log", error="Problem getting commit log", error_msg=e.message, track=self.track)
def __init__(self, config, name, options, index=-1): # pylint: disable=too-many-branches """Init single ssh command""" self.limit_to_hosts = [] self.exclude_hosts = [] self.limited_host_list = [] self.limit_sites = [] self.limit_indexes = [] self.suppress_limit_to_hosts_warnings = True self.use_auth = [] self.name = name self.cmd = None self.use_root = False self.use_app_binary = True self.pre_install = False self.index = index self.only_run_on_init = False self.delay = consts.REMOTE_CMD_RUN_SLEEP_TIMER for option in options: try: config_option = config.get(name, option).strip('"\'') if option == 'limit_to_hosts': self.limit_to_hosts = json.loads(config_option) elif option == 'exclude_hosts': self.exclude_hosts = json.loads(config_option) elif option == 'limit_sites': self.limit_sites = json.loads(config_option) elif option == 'limit_indexes': self.limit_indexes = json.loads(config_option) elif option == 'use_root': self.use_root = config.getboolean(name, option) elif option == 'use_app_binary': self.use_app_binary = config.getboolean(name, option) elif option == 'suppress_limit_to_hosts_warnings': self.suppress_limit_to_hosts_warnings = config.getboolean( name, option) elif option == 'use_auth': self.use_auth = json.loads(config_option) elif option == 'pre_install': self.pre_install = config.getboolean(name, option) elif option == 'only_run_on_init': self.only_run_on_init = config.getboolean(name, option) elif option == 'cmd': self.cmd = config_option elif option == 'delay': self.delay = int(config_option) except Exception as e: logger.errorout("Problem getting option from command conf", name=name, option=option, module=COMMAND_MODULE_INIT, error_msg=str(e))
def build_hostname(naming_format, s_class, host_num): """Build a hostname based on class and num given""" name = render_template( naming_format, { consts.NAME_FORMATTING[0]['name']: s_class, # class consts.NAME_FORMATTING[1]['name']: "0", # site consts.NAME_FORMATTING[2]['name']: host_num # host index }) if len(get_template_vars(name)) > 1: logger.errorout("Host name not templated correctly") return name
def set_commit_id(self, commit_id=None): """Checks out the commit id for the repo """ checkout_id = commit_id if commit_id else self.branch # Already checked out if self.prev_commit == checkout_id: return True cmd = "git checkout {0}".format(checkout_id) output, rc = self.run_command(cmd) if rc > 0: # Corrupted checkout state, try to recover logger.warn("Possible corrupted checkout state", desc="Problem with checkout", error=output, commit_id=checkout_id, path=self.paths['repo_path'], cmd=cmd, track=self.track) # Want to guarantee that the branch is completely reset. git_reset_output, rc = self.run_command( "git reset --hard {0}".format(checkout_id)) #pylint: disable=unused-variable if rc < 1: # Clean up git so there are no untracked files. self.run_command("git clean -fd") if rc > 0: logger.errorout("set_commit_id", desc="Problem setting commit id", error=output, commit_id=checkout_id, path=self.paths['repo_path'], cmd=cmd, track=self.track) self.prev_commit = checkout_id return True
def __init__(self, config, name, options, template_values): """Auth handling""" self.__postfix = "" self.__postfix_filtered = "" self.__postfix_reads = {} self.__inputs = [] self.__delay = consts.REMOTE_AUTH_RUN_SLEEP_TIMER self.__template_values = template_values for option in options: try: config_option = config.get(name, option) if option == 'postfix': self.__prefix = config_option.strip('"\'') self.__postfix_filtered = self.__prefix # Find template values using regex template_groups = re.findall("\\{\\{\\s*\\w+\\s*\\}\\}", self.__prefix, re.DOTALL) if template_groups: auth_kv = {} for template_key in template_groups: field = template_key.strip("{} ") if field not in auth_kv: auth_kv[field] = {"fields": []} if template_key not in auth_kv[field]["fields"]: auth_kv[field]["fields"].append(template_key) # Replace cmd vars with locally generated vars which hide values for k, v in auth_kv.items(): for j2_value in v["fields"]: self.__postfix_filtered = self.__postfix_filtered.replace( j2_value, "$%s" % k) self.__postfix_reads[k] = "{{ %s }}" % k except Exception as e: logger.errorout("Problem getting option from auth", name=name, module=COMMAND_MODULE_INIT, error_msg=str(e))
def get_config(config_file): """Read and get a config object Reads and checks an external configuration file """ config = ConfigParser.ConfigParser(allow_no_value=True) config_fullpath = os.path.abspath(os.path.expandvars(config_file)) # set xform for config otherwise text will be normalized to lowercase config.optionxform = str if not os.path.isfile(config_fullpath): logger.errorout("Commands config file does not exist", path=config_file) try: if not config.read(config_fullpath): logger.errorout("Problem reading command config file") except Exception as e: logger.errorout('Error reading command config', path=config_file, error=e.message) return config
def __init__(self, config, name, options, render_funct, index=-1): """Init single ssh command""" self.limit_to_hosts = [] self.suppress_limit_to_hosts_warnings = False self.use_auth = False self.name = name self.cmd = None self.use_root = False self.use_app_binary = True self.pre_install = False self.index = index self.delay = consts.REMOTE_CMD_RUN_SLEEP_TIMER for option in options: try: config_option = config.get(name, option).strip('"\'') if COMMAND_CLASS_NAME == option: self.limit_to_hosts = json.loads(config_option) elif option == 'use_root': self.use_root = config.getboolean(name, option) elif option == 'use_app_binary': self.use_app_binary = config.getboolean(name, option) elif option == 'suppress_limit_to_hosts_warnings': self.suppress_limit_to_hosts_warnings = config.getboolean( name, option) elif option == 'use_auth': self.use_auth = config.getboolean(name, option) elif option == 'pre_install': self.pre_install = config.getboolean(name, option) elif option == 'cmd': self.cmd = render_funct(config_option) elif option == 'delay': self.delay = int(config_option) except Exception as e: logger.errorout("Problem getting option from command conf", name=name, option=option, module=COMMAND_MODULE_INIT, error_msg=str(e))
def __init__(self, _config, section, skip_check=False): """Init stores vars from a configuration file""" options = self.section_data = _config.options(section) self.data = {'name': section} for seq in consts.DM_COMMANDS_SEQUENCE: self.data[seq] = [] # Checks to make sure all mandatory keys are present if not skip_check: check_options = list(set(DM_MANDATORY_KEYS) - set(options)) if len(check_options) > 0: logger.errorout("key not found in deployment methods", name=section, options=options) for bool_key in DM_BOOL_KEYS: self.data[bool_key] = False self.data['install_ignore'] = "" # Load vars from configuration file for option in options: try: if option in DM_BOOL_KEYS: self.data[option] = _config.getboolean(section, option) else: config_option = _config.get(section, option).strip('"\'') if 'command' in option: self.data[consts.DM_COMMANDS_SEQUENCE[1]].append( config_option) elif 'script' in option: if len(self.data[consts.DM_COMMANDS_SEQUENCE[1]]) < 1: self.data[consts.DM_COMMANDS_SEQUENCE[0]].append( config_option) else: self.data[consts.DM_COMMANDS_SEQUENCE[2]].append( config_option) elif 'install_ignore' in option: self.data[option] = config_option.split(';') elif 'update_method' in option: self.data[option] = config_option # Checking to see if update methods exist if config_option in UPDATE_METHODS or section.startswith( DM_STARTUP_PREFIX): self.data[option] = config_option else: logger.errorout("Update method does not exist", method_used=section, app=config_option) else: self.data[option] = config_option except Exception as e: logger.errorout( "Problem getting option from deployment methods", name=section, option=option, error=e.message)
def load_commands(self): """Load restricted cmd command""" sections = self.config.sections() index = 0 for section in sections: options = self.config.options(section) if section == 'auth': self.__auth = self.render_values( self.config.get(section, 'postfix')) continue self._commands[section] = SshAppCommand(self.config, section, options, self.render_values, index) index += 1 if not set(MANDATORY_COMMAND_STANZAS) <= set(self._commands): logger.errorout("Missing command stanza", needed=MANDATORY_COMMAND_STANZAS, module=COMMAND_MODULE_INIT)
def run(cmd, working_dir=None, dry_run=False): """Runs local cmd command""" cmd_split = shlex.split(cmd) if isinstance(cmd, basestring) else cmd if dry_run: return " ".join(cmd_split), 0 try: p = Popen(cmd_split, shell=False, stderr=STDOUT, stdout=PIPE, cwd=working_dir) communicate = p.communicate() return communicate[0].strip(), p.returncode except OSError as e: logger.errorout("Run OSError", error=e.message) except: # pylint: disable=bare-except logger.errorout("Run Error") return
def get_template_content(path): """Read either yml or json files and store them as dict""" template_dict = {} _filename, file_extension = os.path.splitext(path) file_extension = file_extension.replace('.', '') if file_extension in consts.TEMPLATING_EXTS: try: template_content = {} abs_path = os.path.abspath(os.path.expandvars(path)) with open(abs_path, 'r') as stream: if file_extension in consts.JSON_EXTS: template_content = json.load(stream) #nosec elif file_extension in consts.YMAL_EXTS: template_content = yaml.safe_load(stream) #nosec template_dict.update(template_content) except Exception as e: logger.errorout("Error reading templating file", file=path, error=e.message) else: logger.errorout("No templating file found", file=path) return template_dict
def load_commands(self): """Load restricted cmd command""" sections = self.config.sections() index = 0 for section in sections: options = self.config.options(section) if section.startswith('auth'): self.__auth[section] = SshAppAuth(self.config, section, options, self.template_values) continue self._commands[section] = SshAppCommand(self.config, section, options, index) index += 1 if not set(MANDATORY_COMMAND_STANZAS) <= set(self._commands): logger.errorout("Missing command stanza", needed=MANDATORY_COMMAND_STANZAS, module=COMMAND_MODULE_INIT)
def set_commit_id(self, commit_id=None): """Checks out the commit id for the repo """ checkout_id = commit_id if commit_id else self.branch # Already checked out if self.prev_commit == checkout_id: return True cmd = "git checkout {0}".format(checkout_id) output, rc = helpers.run(cmd, self.paths['repo_path'], self.dryrun) if rc > 0: logger.errorout("set_commit_id", desc="Problem setting commit id", error=output, commit_id=checkout_id, path=self.paths['repo_path'], cmd=cmd, track=self.track) self.prev_commit = checkout_id return True
def check_file(filepath): """Check is file exists""" if not os.path.isfile(filepath): logger.errorout("Can not find file", path=filepath) return True