def get_ssh_client(hostname, ssh_hostname):
"""Tries to create ssh client
Create ssh client based on the username and ssh key
"""
if not CREDS.SSH_KEYFILE:
logger.errorout("ssh_keyfile not set",
module=COMMAND_MODULE_CUSTOM)
retries = 0
while retries < MAX_SSH_RETRIES:
try:
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=ssh_hostname,
username=CREDS.SSH_USER,
port=CREDS.SSH_PORT,
pkey=CREDS.PK,
timeout=CONNECTION_TIMEOUT)
return ssh
except paramiko.BadAuthenticationType:
logger.error("BadAuthenticationType",
hostname=hostname,
module=COMMAND_MODULE_CUSTOM)
return
except paramiko.AuthenticationException:
logger.error("Authentication failed",
hostname=hostname,
module=COMMAND_MODULE_CUSTOM)
return
except paramiko.BadHostKeyException:
logger.error("BadHostKeyException",
fix="Edit known_hosts file to remove the entry",
hostname=hostname,
module=COMMAND_MODULE_CUSTOM)
return
except paramiko.SSHException:
logger.error("SSHException",
hostname=hostname,
module=COMMAND_MODULE_CUSTOM)
return
except Exception as e:
if retries == 0:
logger.error("Problems connecting to host",
hostname=hostname,
module=COMMAND_MODULE_CUSTOM,
error=e.message)
retries += 1
time.sleep(1)
logger.error("Can not connect to host",
hostname=hostname,
module=COMMAND_MODULE_CUSTOM)
return None
def from_params(self,
_app,
_app_clean,
_method,
_commit_id,
_hostname,
_is_firstrun=False):
"""populate application from params"""
self._deployment_method = self.__deployment_mng.get_deployment_method(
_method)
if not self._deployment_method:
logger.errorout("Deployment method for app invalid",
app=_app,
method=_method)
self.app = _app
self.app_clean = _app_clean
self.source_hostname = _hostname
self.method_info = self._deployment_method.data.copy()
self._commit_id = None if _commit_id == 'N/A' or len(
_commit_id) == 0 else _commit_id
self.is_firstrun = _is_firstrun
self.track = self.__repo_mng.track
self.status = consts.META_APP_UNCHANGED
self.updated = False
self.app_creation_datetime = helpers.get_utc()
self.content_type = helpers.get_update_str(False)
return self
def template_directory(app_path, templating_values):
"""Template files
Walks through all the files a directory and templates any jinja2 values
found.
"""
if not check_path(app_path):
logger.errorout("Can not copy location that does not exist",
path=app_path)
tvalues = merge_templates(templating_values)
for path, _dir, files in os.walk(app_path):
# sort files so logs read better and easier to get status
files.sort()
j2_env = Environment(autoescape=True, loader=FileSystemLoader(path))
for filename in files:
# Should not template version file since it may have
# regex commands that can break templating.
if filename.startswith(consts.VERSIONS_FILENAME):
continue
file_path = os.path.join(path, filename)
try:
file_content = j2_env.get_template(filename).render(tvalues)
with open(file_path, 'w') as f:
f.write(file_content)
except Exception as e:
logger.errorout('Error templating file',
file=file_path,
error=e.message)
def pull_repo(self, force=False):
"""Clone repo to specified dir. Delete repo if it currently exist unless reuse.
"""
try:
helpers.create_path(self.paths['absolute_path'], True)
if force:
self.delete_repo()
if not os.path.exists(self.paths['repo_path']):
logger.info("Starting Repo Cloning", track=self.track)
output, rc = helpers.run(
"git clone -b %s %s" % (self.branch, self.url),
self.paths['absolute_path'], self.dryrun)
if rc > 0:
self.delete_repo()
logger.error("Pulling_repo",
error=output,
path=self.paths['repo_path'])
return -1
return 1
else:
return 0
except Exception as e:
logger.errorout("Pulling_repo",
err_msg=e.message,
error="Error pulling repo",
path=self.paths['repo_path'])
def get_commit_log(self):
"""Get the current commit log
"""
try:
log_object = {}
for key, value in COMMIT_KEYS.items():
stdout, _rc = helpers.run(
['git', 'log', '-1',
'--pretty=\'%s\'' % value], self.paths['repo_path'],
self.dryrun)
output = "XXXXX" if self.dryrun else helpers.filter_content(
stdout)
if key in consts.RENAME_COMMIT_LOG_KEYS:
key = consts.RENAME_COMMIT_LOG_KEYS[key]
log_object[key] = output
log_object['project'] = self.project
log_object['reponame'] = self.reponame
return log_object
except Exception as e:
logger.errorout("get_commit_log",
error="Problem getting commit log",
error_msg=e.message,
track=self.track)
def __init__(self, config, name, options, index=-1): # pylint: disable=too-many-branches
"""Init single ssh command"""
self.limit_to_hosts = []
self.exclude_hosts = []
self.limited_host_list = []
self.limit_sites = []
self.limit_indexes = []
self.suppress_limit_to_hosts_warnings = True
self.use_auth = []
self.name = name
self.cmd = None
self.use_root = False
self.use_app_binary = True
self.pre_install = False
self.index = index
self.only_run_on_init = False
self.delay = consts.REMOTE_CMD_RUN_SLEEP_TIMER
for option in options:
try:
config_option = config.get(name, option).strip('"\'')
if option == 'limit_to_hosts':
self.limit_to_hosts = json.loads(config_option)
elif option == 'exclude_hosts':
self.exclude_hosts = json.loads(config_option)
elif option == 'limit_sites':
self.limit_sites = json.loads(config_option)
elif option == 'limit_indexes':
self.limit_indexes = json.loads(config_option)
elif option == 'use_root':
self.use_root = config.getboolean(name, option)
elif option == 'use_app_binary':
self.use_app_binary = config.getboolean(name, option)
elif option == 'suppress_limit_to_hosts_warnings':
self.suppress_limit_to_hosts_warnings = config.getboolean(
name, option)
elif option == 'use_auth':
self.use_auth = json.loads(config_option)
elif option == 'pre_install':
self.pre_install = config.getboolean(name, option)
elif option == 'only_run_on_init':
self.only_run_on_init = config.getboolean(name, option)
elif option == 'cmd':
self.cmd = config_option
elif option == 'delay':
self.delay = int(config_option)
except Exception as e:
logger.errorout("Problem getting option from command conf",
name=name,
option=option,
module=COMMAND_MODULE_INIT,
error_msg=str(e))
def build_hostname(naming_format, s_class, host_num):
"""Build a hostname based on class and num given"""
name = render_template(
naming_format,
{
consts.NAME_FORMATTING[0]['name']: s_class, # class
consts.NAME_FORMATTING[1]['name']: "0", # site
consts.NAME_FORMATTING[2]['name']: host_num # host index
})
if len(get_template_vars(name)) > 1:
logger.errorout("Host name not templated correctly")
return name
def set_commit_id(self, commit_id=None):
"""Checks out the commit id for the repo
"""
checkout_id = commit_id if commit_id else self.branch
# Already checked out
if self.prev_commit == checkout_id:
return True
cmd = "git checkout {0}".format(checkout_id)
output, rc = self.run_command(cmd)
if rc > 0:
# Corrupted checkout state, try to recover
logger.warn("Possible corrupted checkout state",
desc="Problem with checkout",
error=output,
commit_id=checkout_id,
path=self.paths['repo_path'],
cmd=cmd,
track=self.track)
# Want to guarantee that the branch is completely reset.
git_reset_output, rc = self.run_command(
"git reset --hard {0}".format(checkout_id)) #pylint: disable=unused-variable
if rc < 1:
# Clean up git so there are no untracked files.
self.run_command("git clean -fd")
if rc > 0:
logger.errorout("set_commit_id",
desc="Problem setting commit id",
error=output,
commit_id=checkout_id,
path=self.paths['repo_path'],
cmd=cmd,
track=self.track)
self.prev_commit = checkout_id
return True
def __init__(self, config, name, options, template_values):
"""Auth handling"""
self.__postfix = ""
self.__postfix_filtered = ""
self.__postfix_reads = {}
self.__inputs = []
self.__delay = consts.REMOTE_AUTH_RUN_SLEEP_TIMER
self.__template_values = template_values
for option in options:
try:
config_option = config.get(name, option)
if option == 'postfix':
self.__prefix = config_option.strip('"\'')
self.__postfix_filtered = self.__prefix
# Find template values using regex
template_groups = re.findall("\\{\\{\\s*\\w+\\s*\\}\\}",
self.__prefix, re.DOTALL)
if template_groups:
auth_kv = {}
for template_key in template_groups:
field = template_key.strip("{} ")
if field not in auth_kv:
auth_kv[field] = {"fields": []}
if template_key not in auth_kv[field]["fields"]:
auth_kv[field]["fields"].append(template_key)
# Replace cmd vars with locally generated vars which hide values
for k, v in auth_kv.items():
for j2_value in v["fields"]:
self.__postfix_filtered = self.__postfix_filtered.replace(
j2_value, "$%s" % k)
self.__postfix_reads[k] = "{{ %s }}" % k
except Exception as e:
logger.errorout("Problem getting option from auth",
name=name,
module=COMMAND_MODULE_INIT,
error_msg=str(e))
def get_config(config_file):
"""Read and get a config object
Reads and checks an external configuration file
"""
config = ConfigParser.ConfigParser(allow_no_value=True)
config_fullpath = os.path.abspath(os.path.expandvars(config_file))
# set xform for config otherwise text will be normalized to lowercase
config.optionxform = str
if not os.path.isfile(config_fullpath):
logger.errorout("Commands config file does not exist",
path=config_file)
try:
if not config.read(config_fullpath):
logger.errorout("Problem reading command config file")
except Exception as e:
logger.errorout('Error reading command config',
path=config_file,
error=e.message)
return config
def __init__(self, config, name, options, render_funct, index=-1):
"""Init single ssh command"""
self.limit_to_hosts = []
self.suppress_limit_to_hosts_warnings = False
self.use_auth = False
self.name = name
self.cmd = None
self.use_root = False
self.use_app_binary = True
self.pre_install = False
self.index = index
self.delay = consts.REMOTE_CMD_RUN_SLEEP_TIMER
for option in options:
try:
config_option = config.get(name, option).strip('"\'')
if COMMAND_CLASS_NAME == option:
self.limit_to_hosts = json.loads(config_option)
elif option == 'use_root':
self.use_root = config.getboolean(name, option)
elif option == 'use_app_binary':
self.use_app_binary = config.getboolean(name, option)
elif option == 'suppress_limit_to_hosts_warnings':
self.suppress_limit_to_hosts_warnings = config.getboolean(
name, option)
elif option == 'use_auth':
self.use_auth = config.getboolean(name, option)
elif option == 'pre_install':
self.pre_install = config.getboolean(name, option)
elif option == 'cmd':
self.cmd = render_funct(config_option)
elif option == 'delay':
self.delay = int(config_option)
except Exception as e:
logger.errorout("Problem getting option from command conf",
name=name,
option=option,
module=COMMAND_MODULE_INIT,
error_msg=str(e))
def __init__(self, _config, section, skip_check=False):
"""Init stores vars from a configuration file"""
options = self.section_data = _config.options(section)
self.data = {'name': section}
for seq in consts.DM_COMMANDS_SEQUENCE:
self.data[seq] = []
# Checks to make sure all mandatory keys are present
if not skip_check:
check_options = list(set(DM_MANDATORY_KEYS) - set(options))
if len(check_options) > 0:
logger.errorout("key not found in deployment methods",
name=section,
options=options)
for bool_key in DM_BOOL_KEYS:
self.data[bool_key] = False
self.data['install_ignore'] = ""
# Load vars from configuration file
for option in options:
try:
if option in DM_BOOL_KEYS:
self.data[option] = _config.getboolean(section, option)
else:
config_option = _config.get(section, option).strip('"\'')
if 'command' in option:
self.data[consts.DM_COMMANDS_SEQUENCE[1]].append(
config_option)
elif 'script' in option:
if len(self.data[consts.DM_COMMANDS_SEQUENCE[1]]) < 1:
self.data[consts.DM_COMMANDS_SEQUENCE[0]].append(
config_option)
else:
self.data[consts.DM_COMMANDS_SEQUENCE[2]].append(
config_option)
elif 'install_ignore' in option:
self.data[option] = config_option.split(';')
elif 'update_method' in option:
self.data[option] = config_option
# Checking to see if update methods exist
if config_option in UPDATE_METHODS or section.startswith(
DM_STARTUP_PREFIX):
self.data[option] = config_option
else:
logger.errorout("Update method does not exist",
method_used=section,
app=config_option)
else:
self.data[option] = config_option
except Exception as e:
logger.errorout(
"Problem getting option from deployment methods",
name=section,
option=option,
error=e.message)
def load_commands(self):
"""Load restricted cmd command"""
sections = self.config.sections()
index = 0
for section in sections:
options = self.config.options(section)
if section == 'auth':
self.__auth = self.render_values(
self.config.get(section, 'postfix'))
continue
self._commands[section] = SshAppCommand(self.config, section,
options,
self.render_values, index)
index += 1
if not set(MANDATORY_COMMAND_STANZAS) <= set(self._commands):
logger.errorout("Missing command stanza",
needed=MANDATORY_COMMAND_STANZAS,
module=COMMAND_MODULE_INIT)
def run(cmd, working_dir=None, dry_run=False):
"""Runs local cmd command"""
cmd_split = shlex.split(cmd) if isinstance(cmd, basestring) else cmd
if dry_run:
return " ".join(cmd_split), 0
try:
p = Popen(cmd_split,
shell=False,
stderr=STDOUT,
stdout=PIPE,
cwd=working_dir)
communicate = p.communicate()
return communicate[0].strip(), p.returncode
except OSError as e:
logger.errorout("Run OSError", error=e.message)
except: # pylint: disable=bare-except
logger.errorout("Run Error")
return
def get_template_content(path):
"""Read either yml or json files and store them as dict"""
template_dict = {}
_filename, file_extension = os.path.splitext(path)
file_extension = file_extension.replace('.', '')
if file_extension in consts.TEMPLATING_EXTS:
try:
template_content = {}
abs_path = os.path.abspath(os.path.expandvars(path))
with open(abs_path, 'r') as stream:
if file_extension in consts.JSON_EXTS:
template_content = json.load(stream) #nosec
elif file_extension in consts.YMAL_EXTS:
template_content = yaml.safe_load(stream) #nosec
template_dict.update(template_content)
except Exception as e:
logger.errorout("Error reading templating file",
file=path,
error=e.message)
else:
logger.errorout("No templating file found", file=path)
return template_dict
def load_commands(self):
"""Load restricted cmd command"""
sections = self.config.sections()
index = 0
for section in sections:
options = self.config.options(section)
if section.startswith('auth'):
self.__auth[section] = SshAppAuth(self.config, section,
options,
self.template_values)
continue
self._commands[section] = SshAppCommand(self.config, section,
options, index)
index += 1
if not set(MANDATORY_COMMAND_STANZAS) <= set(self._commands):
logger.errorout("Missing command stanza",
needed=MANDATORY_COMMAND_STANZAS,
module=COMMAND_MODULE_INIT)
def set_commit_id(self, commit_id=None):
"""Checks out the commit id for the repo
"""
checkout_id = commit_id if commit_id else self.branch
# Already checked out
if self.prev_commit == checkout_id:
return True
cmd = "git checkout {0}".format(checkout_id)
output, rc = helpers.run(cmd, self.paths['repo_path'], self.dryrun)
if rc > 0:
logger.errorout("set_commit_id",
desc="Problem setting commit id",
error=output,
commit_id=checkout_id,
path=self.paths['repo_path'],
cmd=cmd,
track=self.track)
self.prev_commit = checkout_id
return True
def check_file(filepath):
"""Check is file exists"""
if not os.path.isfile(filepath):
logger.errorout("Can not find file", path=filepath)
return True
