def _update_repo_metadata_after_bad_try(self, cmd, code, out):
name = self.get_package_name_with_version()
repo_update_cmd = self.get_repo_update_cmd()
Logger.info("Execution of '%s' returned %d. %s" % (shell.string_cmd_from_args_list(cmd), code, out))
Logger.info("Failed to install package %s. Executing '%s'" % (name, shell.string_cmd_from_args_list(repo_update_cmd)))
code, out = shell.call(repo_update_cmd, sudo=True, logoutput=self.get_logoutput())
if code:
Logger.info("Execution of '%s' returned %d. %s" % (repo_update_cmd, code, out))
Logger.info("Retrying to install package %s after %d seconds" % (name, self.resource.retry_sleep))
def remove_package(self, name):
if self._check_existence(name):
cmd = REMOVE_CMD[self.get_logoutput()] + [name]
Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput())
else:
Logger.info("Skipping removal of non-existing package %s" % (name))
def install_package(self,
name,
use_repos={},
skip_repos=[],
is_upgrade=False):
if is_upgrade or use_repos or not self._check_existence(name):
cmd = INSTALL_CMD[self.get_logoutput()]
use_repos = use_repos.keys()
if use_repos:
active_base_repos = self.get_active_base_repos()
if 'base' in use_repos:
# Remove 'base' from use_repos list
use_repos = filter(lambda x: x != 'base', use_repos)
use_repos.extend(active_base_repos)
use_repos_options = []
for repo in sorted(use_repos):
use_repos_options = use_repos_options + ['--repo', repo]
cmd = cmd + use_repos_options
cmd = cmd + [name]
Logger.info("Installing package %s ('%s')" %
(name, string_cmd_from_args_list(cmd)))
self.checked_call_with_retries(cmd,
sudo=True,
logoutput=self.get_logoutput())
else:
Logger.info("Skipping installation of existing package %s" %
(name))
def get_user_call_output(command, user, is_checked_call=True, **call_kwargs):
"""
This function eliminates only output of command inside the su, ignoring the su ouput itself.
This is useful since some users have motd messages setup by default on su -l.
@return: code, stdout, stderr
"""
command_string = shell.string_cmd_from_args_list(command) if isinstance(command, (list, tuple)) else command
out_files = []
try:
out_files.append(tempfile.NamedTemporaryFile())
out_files.append(tempfile.NamedTemporaryFile())
# other user should be able to write to it
for f in out_files:
os.chmod(f.name, 0666)
command_string += " 1>" + out_files[0].name
command_string += " 2>" + out_files[1].name
func = shell.checked_call if is_checked_call else shell.call
func_result = func(shell.as_user(command_string, user), **call_kwargs)
files_output = []
for f in out_files:
files_output.append(f.read())
return func_result[0], files_output[0], files_output[1]
finally:
for f in out_files:
f.close()
def gpsshify(command, host=None, hostfile=None, args=None):
"""Return a gpssh command which will run the command on the specified remote machine in the cluster.
NOTE: User must be specified when running command using Execute! Even if user is root.
command -- Command or commands which will be run on remote machines in the cluster.
host -- A remote host to run the command on.
hostfile -- The path to a file containing a list of hosts to run the command on.
args -- Additional arguments to append to the gpssh command.
"""
if host == None and hostfile == None:
raise ValueError('Either host or hostfile must be given')
if isinstance(command, (list, tuple)):
command = string_cmd_from_args_list(command)
arguments = []
if host != None:
arguments.append('-h "%s"' % host)
if hostfile != None:
arguments.append('-f "%s"' % hostfile)
if args != None:
arguments.append(args)
arguments = " ".join(arguments)
return format(
dedent("""
cat <<EOF | gpssh {arguments}
{command}
EOF
""".rstrip()))
def run_command(self,
target,
operation,
method='POST',
assertable_result=True,
file_to_put=None,
ignore_status_codes=[],
**kwargs):
"""
assertable_result - some POST requests return '{"boolean":false}' or '{"boolean":true}'
depending on if query was successful or not, we can assert this for them
"""
target = HdfsResourceProvider.parse_path(target)
url = format(
"{address}/webhdfs/v1{target}?op={operation}&user.name={run_user}",
address=self.address,
run_user=self.run_user)
for k, v in kwargs.iteritems():
url = format("{url}&{k}={v}")
if file_to_put and not os.path.exists(file_to_put):
raise Fail(format("File {file_to_put} is not found."))
cmd = ["curl", "-sS", "-L", "-w", "%{http_code}", "-X", method]
if file_to_put:
cmd += [
"--data-binary", "@" + file_to_put, "-H",
"Content-Type: application/octet-stream"
]
if self.security_enabled:
cmd += ["--negotiate", "-u", ":"]
if self.is_https_enabled:
cmd += ["-k"]
cmd.append(url)
_, out, err = get_user_call_output(cmd,
user=self.run_user,
logoutput=self.logoutput,
quiet=False)
status_code = out[-3:]
out = out[:-3] # remove last line from output which is status code
try:
result_dict = json.loads(out)
except ValueError:
result_dict = out
if status_code not in WebHDFSUtil.valid_status_codes + ignore_status_codes or assertable_result and result_dict and not result_dict[
'boolean']:
formatted_output = json.dumps(result_dict, indent=2) if isinstance(
result_dict, dict) else result_dict
formatted_output = err + "\n" + formatted_output
err_msg = "Execution of '%s' returned status_code=%s. %s" % (
shell.string_cmd_from_args_list(cmd), status_code,
formatted_output)
raise Fail(err_msg)
return result_dict
def remove_package(self, name):
if self._check_existence(name):
cmd = REMOVE_CMD[self.get_logoutput()] + [name]
Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput())
else:
Logger.info("Skipping removal of non-existing package %s" % (name))
def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False):
if is_upgrade or use_repos or not self._check_existence(name):
cmd = INSTALL_CMD[self.get_logoutput()]
copied_sources_files = []
is_tmp_dir_created = False
if use_repos:
is_tmp_dir_created = True
apt_sources_list_tmp_dir = tempfile.mkdtemp(suffix="-ambari-apt-sources-d")
Logger.info("Temporal sources directory was created: %s" % apt_sources_list_tmp_dir)
if 'base' not in use_repos:
cmd = cmd + ['-o', 'Dir::Etc::SourceList=%s' % EMPTY_FILE]
for repo in use_repos:
if repo != 'base':
new_sources_file = os.path.join(apt_sources_list_tmp_dir, repo + '.list')
Logger.info("Temporal sources file will be copied: %s" % new_sources_file)
sudo.copy(os.path.join(APT_SOURCES_LIST_DIR, repo + '.list'), new_sources_file)
copied_sources_files.append(new_sources_file)
cmd = cmd + ['-o', 'Dir::Etc::SourceParts=%s' % apt_sources_list_tmp_dir]
cmd = cmd + [name]
Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
code, out = self.call_with_retries(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput())
if self.is_locked_output(out):
err_msg = Logger.filter_text("Execution of '%s' returned %d. %s" % (cmd, code, out))
raise Fail(err_msg)
# apt-get update wasn't done too long maybe?
if code:
Logger.info("Execution of '%s' returned %d. %s" % (cmd, code, out))
Logger.info("Failed to install package %s. Executing `%s`" % (name, string_cmd_from_args_list(REPO_UPDATE_CMD)))
code, out = self.call_with_retries(REPO_UPDATE_CMD, sudo=True, logoutput=self.get_logoutput())
if code:
Logger.info("Execution of '%s' returned %d. %s" % (REPO_UPDATE_CMD, code, out))
Logger.info("Retrying to install package %s" % (name))
self.checked_call_with_retries(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput())
if is_tmp_dir_created:
for temporal_sources_file in copied_sources_files:
Logger.info("Removing temporal sources file: %s" % temporal_sources_file)
os.remove(temporal_sources_file)
Logger.info("Removing temporal sources directory: %s" % apt_sources_list_tmp_dir)
os.rmdir(apt_sources_list_tmp_dir)
else:
Logger.info("Skipping installation of existing package %s" % (name))
def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False):
if is_upgrade or use_repos or not self._check_existence(name):
cmd = INSTALL_CMD[self.get_logoutput()]
copied_sources_files = []
is_tmp_dir_created = False
if use_repos:
is_tmp_dir_created = True
apt_sources_list_tmp_dir = tempfile.mkdtemp(suffix="-ambari-apt-sources-d")
Logger.info("Temporal sources directory was created: %s" % apt_sources_list_tmp_dir)
if 'base' not in use_repos:
cmd = cmd + ['-o', 'Dir::Etc::SourceList=%s' % EMPTY_FILE]
for repo in use_repos:
if repo != 'base':
new_sources_file = os.path.join(apt_sources_list_tmp_dir, repo + '.list')
Logger.info("Temporal sources file will be copied: %s" % new_sources_file)
sudo.copy(os.path.join(APT_SOURCES_LIST_DIR, repo + '.list'), new_sources_file)
copied_sources_files.append(new_sources_file)
cmd = cmd + ['-o', 'Dir::Etc::SourceParts=%s' % apt_sources_list_tmp_dir]
cmd = cmd + [name]
Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
code, out = self.call_until_not_locked(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput())
if self.is_locked_output(out):
err_msg = Logger.filter_text("Execution of '%s' returned %d. %s" % (cmd, code, out))
raise Fail(err_msg)
# apt-get update wasn't done too long maybe?
if code:
Logger.info("Execution of '%s' returned %d. %s" % (cmd, code, out))
Logger.info("Failed to install package %s. Executing `%s`" % (name, string_cmd_from_args_list(REPO_UPDATE_CMD)))
code, out = self.call_until_not_locked(REPO_UPDATE_CMD, sudo=True, logoutput=self.get_logoutput())
if code:
Logger.info("Execution of '%s' returned %d. %s" % (REPO_UPDATE_CMD, code, out))
Logger.info("Retrying to install package %s" % (name))
self.checked_call_until_not_locked(cmd, sudo=True, env=INSTALL_CMD_ENV, logoutput=self.get_logoutput())
if is_tmp_dir_created:
for temporal_sources_file in copied_sources_files:
Logger.info("Removing temporal sources file: %s" % temporal_sources_file)
os.remove(temporal_sources_file)
Logger.info("Removing temporal sources directory: %s" % apt_sources_list_tmp_dir)
os.rmdir(apt_sources_list_tmp_dir)
else:
Logger.info("Skipping installation of existing package %s" % (name))
def install_package(self,
name,
use_repos={},
skip_repos=set(),
is_upgrade=False):
if is_upgrade or use_repos or not self._check_existence(name):
cmd = INSTALL_CMD[self.get_logoutput()]
copied_sources_files = []
is_tmp_dir_created = False
if use_repos:
if 'base' in use_repos:
use_repos = set(
[v for k, v in use_repos.items() if k != 'base'])
else:
cmd = cmd + ['-o', 'Dir::Etc::SourceList=%s' % EMPTY_FILE]
use_repos = set(use_repos.values())
if use_repos:
is_tmp_dir_created = True
apt_sources_list_tmp_dir = tempfile.mkdtemp(
suffix="-ambari-apt-sources-d")
Logger.info("Temporary sources directory was created: %s" %
apt_sources_list_tmp_dir)
for repo in use_repos:
new_sources_file = os.path.join(
apt_sources_list_tmp_dir, repo + '.list')
Logger.info(
"Temporary sources file will be copied: %s" %
new_sources_file)
sudo.copy(
os.path.join(APT_SOURCES_LIST_DIR, repo + '.list'),
new_sources_file)
copied_sources_files.append(new_sources_file)
cmd = cmd + [
'-o',
'Dir::Etc::SourceParts=%s' % apt_sources_list_tmp_dir
]
cmd = cmd + [name]
Logger.info("Installing package %s ('%s')" %
(name, string_cmd_from_args_list(cmd)))
self.checked_call_with_retries(cmd,
sudo=True,
env=INSTALL_CMD_ENV,
logoutput=self.get_logoutput())
if is_tmp_dir_created:
for temporary_sources_file in copied_sources_files:
Logger.info("Removing temporary sources file: %s" %
temporary_sources_file)
os.remove(temporary_sources_file)
Logger.info("Removing temporary sources directory: %s" %
apt_sources_list_tmp_dir)
os.rmdir(apt_sources_list_tmp_dir)
else:
Logger.info("Skipping installation of existing package %s" %
(name))
def get_user_call_output(command,
user,
quiet=False,
is_checked_call=True,
**call_kwargs):
"""
This function eliminates only output of command inside the su, ignoring the su ouput itself.
This is useful since some users have motd messages setup by default on su -l.
@return: code, stdout, stderr
"""
command_string = shell.string_cmd_from_args_list(command) if isinstance(
command, (list, tuple)) else command
out_files = []
try:
out_files.append(tempfile.NamedTemporaryFile())
out_files.append(tempfile.NamedTemporaryFile())
# other user should be able to write to it
for f in out_files:
os.chmod(f.name, 0666)
command_string += " 1>" + out_files[0].name
command_string += " 2>" + out_files[1].name
code, _ = shell.call(shell.as_user(command_string, user),
quiet=quiet,
**call_kwargs)
files_output = []
for f in out_files:
files_output.append(f.read().decode("utf-8").strip('\n'))
if code:
all_output = files_output[1] + '\n' + files_output[0]
err_msg = Logger.filter_text(
("Execution of '%s' returned %d. %s") %
(command_string, code, all_output))
if is_checked_call:
raise Fail(err_msg)
else:
Logger.warning(err_msg)
result = code, files_output[0], files_output[1]
caller_filename = sys._getframe(1).f_code.co_filename
is_internal_call = shell.NOT_LOGGED_FOLDER in caller_filename
if quiet == False or (quiet == None and not is_internal_call):
log_msg = "{0} returned {1}".format(get_user_call_output.__name__,
result)
Logger.info(log_msg)
return result
finally:
for f in out_files:
f.close()
def remove_package(self, name, ignore_dependencies=False):
if self._check_existence(name):
if ignore_dependencies:
cmd = REMOVE_WITHOUT_DEPENDENCIES_CMD + [name]
else:
cmd = REMOVE_CMD[self.get_logoutput()] + [name]
Logger.info("Removing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput())
else:
Logger.info("Skipping removal of non-existing package %s" % (name))
def install_package(self, name, use_repos=[]):
if not self._check_existence(name) or use_repos:
cmd = INSTALL_CMD[self.get_logoutput()]
if use_repos:
enable_repo_option = '--enablerepo=' + ",".join(use_repos)
cmd = cmd + ['--disablerepo=*', enable_repo_option]
cmd = cmd + [name]
Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput())
else:
Logger.info("Skipping installing existent package %s" % (name))
def install_package(self, name, use_repos=[]):
if not self._check_existence(name) or use_repos:
cmd = INSTALL_CMD[self.get_logoutput()]
if use_repos:
enable_repo_option = "--enablerepo=" + ",".join(use_repos)
cmd = cmd + ["--disablerepo=*", enable_repo_option]
cmd = cmd + [name]
Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput())
else:
Logger.info("Skipping installing existent package %s" % (name))
def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False):
if is_upgrade or use_repos or not self._check_existence(name):
cmd = INSTALL_CMD[self.get_logoutput()]
if use_repos:
enable_repo_option = '--enablerepo=' + ",".join(use_repos)
disable_repo_option = '--disablerepo=' + "*,".join(skip_repos)
cmd = cmd + [disable_repo_option, enable_repo_option]
cmd = cmd + [name]
Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput())
else:
Logger.info("Skipping installation of existing package %s" % (name))
def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False):
if is_upgrade or use_repos or not self._check_existence(name):
cmd = INSTALL_CMD[self.get_logoutput()]
if use_repos:
enable_repo_option = '--enablerepo=' + ",".join(use_repos)
disable_repo_option = '--disablerepo=' + "*,".join(skip_repos)
cmd = cmd + [disable_repo_option, enable_repo_option]
cmd = cmd + [name]
Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput())
else:
Logger.info("Skipping installation of existing package %s" % (name))
def get_user_call_output(command,
user,
quiet=False,
is_checked_call=True,
**call_kwargs):
"""
This function eliminates only output of command inside the su, ignoring the su ouput itself.
This is useful since some users have motd messages setup by default on su -l.
@return: code, stdout, stderr
"""
command_string = shell.string_cmd_from_args_list(command) if isinstance(
command, (list, tuple)) else command
out_files = []
try:
out_files.append(tempfile.NamedTemporaryFile())
out_files.append(tempfile.NamedTemporaryFile())
# other user should be able to write to it
for f in out_files:
os.chmod(f.name, 0666)
command_string += " 1>" + out_files[0].name
command_string += " 2>" + out_files[1].name
code, _ = shell.call(shell.as_user(command_string, user),
quiet=quiet,
**call_kwargs)
files_output = []
for f in out_files:
files_output.append(f.read().strip('\n'))
if code:
all_output = files_output[1] + '\n' + files_output[0]
err_msg = Logger.filter_text(
("Execution of '%s' returned %d. %s") %
(command_string, code, all_output))
if is_checked_call:
raise Fail(err_msg)
else:
Logger.warning(err_msg)
return code, files_output[0], files_output[1]
finally:
for f in out_files:
f.close()
def remove_package(self, name, ignore_dependencies=False):
if self._check_existence(name):
if ignore_dependencies:
cmd = REMOVE_WITHOUT_DEPENDENCIES_CMD + [
name, name
] # have to specify name twice: one for --ignore-depends, one for --remove
else:
cmd = REMOVE_CMD[self.get_logoutput()] + [name]
Logger.info("Removing package %s ('%s')" %
(name, string_cmd_from_args_list(cmd)))
self.checked_call_with_retries(cmd,
sudo=True,
logoutput=self.get_logoutput())
else:
Logger.info("Skipping removal of non-existing package %s" % (name))
def install_package(self, name, use_repos=[]):
if not self._check_existence(name) or use_repos:
cmd = INSTALL_CMD[self.get_logoutput()]
if use_repos:
active_base_repos = get_active_base_repos()
if 'base' in use_repos:
use_repos = filter(lambda x: x != 'base', use_repos)
use_repos.extend(active_base_repos)
use_repos_options = []
for repo in use_repos:
use_repos_options = use_repos_options + ['--repo', repo]
cmd = cmd + use_repos_options
cmd = cmd + [name]
Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput())
else:
Logger.info("Skipping installing existent package %s" % (name))
def install_package(self, name, use_repos=[]):
if not self._check_existence(name) or use_repos:
cmd = INSTALL_CMD[self.get_logoutput()]
if use_repos:
active_base_repos = get_active_base_repos()
if 'base' in use_repos:
use_repos = filter(lambda x: x != 'base', use_repos)
use_repos.extend(active_base_repos)
use_repos_options = []
for repo in use_repos:
use_repos_options = use_repos_options + ['--repo', repo]
cmd = cmd + use_repos_options
cmd = cmd + [name]
Logger.info("Installing package %s ('%s')" %
(name, string_cmd_from_args_list(cmd)))
shell.checked_call(cmd, sudo=True, logoutput=self.get_logoutput())
else:
Logger.info("Skipping installing existent package %s" % (name))
def install_package(self, name, use_repos=[], skip_repos=[], is_upgrade=False):
if is_upgrade or use_repos or not self._check_existence(name):
cmd = INSTALL_CMD[self.get_logoutput()]
if use_repos:
active_base_repos = self.get_active_base_repos()
if 'base' in use_repos:
# Remove 'base' from use_repos list
use_repos = filter(lambda x: x != 'base', use_repos)
use_repos.extend(active_base_repos)
use_repos_options = []
for repo in use_repos:
use_repos_options = use_repos_options + ['--repo', repo]
cmd = cmd + use_repos_options
cmd = cmd + [name]
Logger.info("Installing package %s ('%s')" % (name, string_cmd_from_args_list(cmd)))
self.checked_call_with_retries(cmd, sudo=True, logoutput=self.get_logoutput())
else:
Logger.info("Skipping installation of existing package %s" % (name))
def get_user_call_output(command, user, quiet=False, is_checked_call=True, **call_kwargs):
"""
This function eliminates only output of command inside the su, ignoring the su ouput itself.
This is useful since some users have motd messages setup by default on su -l.
@return: code, stdout, stderr
"""
command_string = shell.string_cmd_from_args_list(command) if isinstance(command, (list, tuple)) else command
out_files = []
try:
out_files.append(tempfile.NamedTemporaryFile())
out_files.append(tempfile.NamedTemporaryFile())
# other user should be able to write to it
for f in out_files:
os.chmod(f.name, 0666)
command_string += " 1>" + out_files[0].name
command_string += " 2>" + out_files[1].name
code, _ = shell.call(shell.as_user(command_string, user), quiet=quiet, **call_kwargs)
files_output = []
for f in out_files:
files_output.append(f.read().strip('\n'))
if code:
all_output = files_output[1] + '\n' + files_output[0]
err_msg = Logger.filter_text(("Execution of '%s' returned %d. %s") % (command_string, code, all_output))
if is_checked_call:
raise Fail(err_msg)
else:
Logger.warning(err_msg)
return code, files_output[0], files_output[1]
finally:
for f in out_files:
f.close()
def run_command(self, target, operation, method='POST', assertable_result=True, file_to_put=None, ignore_status_codes=[], **kwargs):
"""
assertable_result - some POST requests return '{"boolean":false}' or '{"boolean":true}'
depending on if query was successful or not, we can assert this for them
"""
target = HdfsResourceProvider.parse_path(target)
url = format("{address}/webhdfs/v1{target}?op={operation}&user.name={run_user}", address=self.address, run_user=self.run_user)
for k,v in kwargs.iteritems():
url = format("{url}&{k}={v}")
if file_to_put and not os.path.exists(file_to_put):
raise Fail(format("File {file_to_put} is not found."))
cmd = ["curl", "-sS","-L", "-w", "%{http_code}", "-X", method]
if file_to_put:
cmd += ["-T", file_to_put]
if self.security_enabled:
cmd += ["--negotiate", "-u", ":"]
if self.is_https_enabled:
cmd += ["-k"]
cmd.append(url)
_, out, err = get_user_call_output(cmd, user=self.run_user, logoutput=self.logoutput, quiet=False)
status_code = out[-3:]
out = out[:-3] # remove last line from output which is status code
try:
result_dict = json.loads(out)
except ValueError:
result_dict = out
if status_code not in WebHDFSUtil.valid_status_codes+ignore_status_codes or assertable_result and result_dict and not result_dict['boolean']:
formatted_output = json.dumps(result_dict, indent=2) if isinstance(result_dict, dict) else result_dict
formatted_output = err + "\n" + formatted_output
err_msg = "Execution of '%s' returned status_code=%s. %s" % (shell.string_cmd_from_args_list(cmd), status_code, formatted_output)
raise Fail(err_msg)
return result_dict
