def check_partial_install(self):
"""
If an installation did not complete successfully, check if installation was partially complete and
log the partially completed version to REPO_VERSION_HISTORY_FILE.
:return:
"""
Logger.info(
"Installation of packages failed. Checking if installation was partially complete"
)
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_stack_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
# Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE
partial_install_version = next(iter(deltas)).strip()
write_actual_version_to_history_file(normalized_repo_version,
partial_install_version)
Logger.info("Version {0} was partially installed. ".format(
partial_install_version))
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is.
"""
# If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
# to correct value if it is not
self.actual_version = None
self.repo_version_with_build_number = None
if self.repository_version:
m = re.search("[\d\.]+-\d+", self.repository_version)
if m:
# Contains a build number
self.repo_version_with_build_number = self.repository_version
self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far.
self.put_structured_out(self.structured_output)
Logger.info("Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version, self.actual_version)
Logger.info(
"Found actual version {0} by checking the delta between versions before and after installing packages".format(
self.actual_version))
else:
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
Logger.info("Cannot determine actual version installed by checking the delta between versions "
"before and after installing package")
Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed")
self.actual_version = self.find_best_fit_version(new_versions, self.repository_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version))
else:
msg = "Could not determine actual version installed. Try reinstalling packages again."
raise Fail(msg)
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is.
"""
# If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
# to correct value if it is not
self.actual_version = None
self.repo_version_with_build_number = None
if self.repository_version:
m = re.search("[\d\.]+-\d+", self.repository_version)
if m:
# Contains a build number
self.repo_version_with_build_number = self.repository_version
self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far.
self.put_structured_out(self.structured_output)
Logger.info("Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_stack_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version, self.actual_version)
Logger.info(
"Found actual version {0} by checking the delta between versions before and after installing packages".format(
self.actual_version))
else:
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
Logger.info("Cannot determine actual version installed by checking the delta between versions "
"before and after installing package")
Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed")
self.actual_version = self.find_best_fit_version(new_versions, self.repository_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version))
else:
msg = "Could not determine actual version installed. Try reinstalling packages again."
raise Fail(msg)
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is, in order to save it.
"""
Logger.info(
"Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions()
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get HDP version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version,
self.actual_version)
else:
Logger.info(
"Cannot determine a new actual version installed by using the delta method."
)
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, it is important to search the
# repo version history file to determine if we previously did write an actual_version.
self.actual_version = read_actual_version_from_history_file(
normalized_repo_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info(
"Found actual version {0} by parsing file {1}".format(
self.actual_version, REPO_VERSION_HISTORY_FILE))
elif self.repo_version_with_build_number is None:
# It's likely that this host does not have any Stack Components installed, so only contains AMS.
# So just use repo version value provided by server (we already put it to structured output)
if not os.path.exists(self.stack_root_folder):
# Special case when this host does not contain any HDP components, but still contains other components like AMS.
msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \
"Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder)
Logger.info(msg)
else:
msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \
"Try reinstalling packages again.".format(self.stack_root_folder)
raise Fail(msg)
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is, in order to save it.
"""
Logger.info(
"Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions()
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get HDP version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version,
self.actual_version)
else:
Logger.info(
"Cannot determine a new actual version installed by using the delta method."
)
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, it is important to search the
# repo version history file to determine if we previously did write an actual_version.
self.actual_version = read_actual_version_from_history_file(
normalized_repo_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info(
"Found actual version {0} by parsing file {1}".format(
self.actual_version, REPO_VERSION_HISTORY_FILE))
elif self.repo_version_with_build_number is None:
msg = "Could not determine actual version installed. Try reinstalling packages again."
raise Fail(msg)
def check_partial_install(self):
"""
If an installation did not complete successfully, check if installation was partially complete and
log the partially completed version to REPO_VERSION_HISTORY_FILE.
:return:
"""
Logger.info("Installation of packages failed. Checking if installation was partially complete")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
# Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE
partial_install_version = next(iter(deltas)).strip()
write_actual_version_to_history_file(normalized_repo_version, partial_install_version)
Logger.info("Version {0} was partially installed. ".format(partial_install_version))
def test_read_and_write_repo_version_history(self):
f, filename = tempfile.mkstemp()
try:
# Check read of empty file
repo_version_history.REPO_VERSION_HISTORY_FILE = filename
repo_version_history.Logger = logging.getLogger()
result = repo_version_history.read_actual_version_from_history_file('2.3.2.0')
self.assertEquals(result, None)
# Check read of single value
repo_version_history.write_actual_version_to_history_file('2.3.2.0', '2.3.2.0-210')
result = repo_version_history.read_actual_version_from_history_file('2.3.2.0')
self.assertEquals(result, '2.3.2.0-210')
# Check read after update
repo_version_history.write_actual_version_to_history_file('2.3.2.0', '2.3.2.0-2716')
result = repo_version_history.read_actual_version_from_history_file('2.3.2.0')
self.assertEquals(result, '2.3.2.0-2716')
# Check read after update
repo_version_history.write_actual_version_to_history_file('2.3.2.0', '2.3.2.0-2758')
result = repo_version_history.read_actual_version_from_history_file('2.3.2.0')
self.assertEquals(result, '2.3.2.0-2758')
# Check read after writing down version for another stack
repo_version_history.write_actual_version_to_history_file('2.3.1.0', '2.3.1.0-27')
result = repo_version_history.read_actual_version_from_history_file('2.3.1.0')
self.assertEquals(result, '2.3.1.0-27')
result = repo_version_history.read_actual_version_from_history_file('2.3.2.0')
self.assertEquals(result, '2.3.2.0-2758')
# Check read of another stack
result = repo_version_history.read_actual_version_from_history_file('2.3.0.0')
self.assertEquals(result, None)
finally:
os.unlink(filename)
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is.
"""
# If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
# to correct value if it is not
self.actual_version = None
self.repo_version_with_build_number = None
if self.repository_version:
m = re.search("[\d\.]+-\d+", self.repository_version)
if m:
# Contains a build number
self.repo_version_with_build_number = self.repository_version
self.structured_output[
'actual_version'] = self.repo_version_with_build_number # This is the best value known so far.
self.put_structured_out(self.structured_output)
Logger.info(
"Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version,
self.actual_version)
Logger.info(
"Found actual version {0} by checking the delta between versions before and after installing packages"
.format(self.actual_version))
else:
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
Logger.info(
"Cannot determine actual version installed by checking the delta between versions "
"before and after installing package")
Logger.info(
"Will try to find for the actual version by searching for best possible match in the list of versions installed"
)
self.actual_version = self.find_best_fit_version(
new_versions, self.repository_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info(
"Found actual version {0} by searching for best possible match"
.format(self.actual_version))
else:
# It's likely that this host does not have any Stack Components installed, so only contains AMS.
# So just use repo version value provided by server (we already put it to structured output)
if not os.path.exists(self.stack_root_folder):
# Special case when this host does not contain any HDP components, but still contains other components like AMS.
msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \
"Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder)
Logger.info(msg)
else:
msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \
"Try reinstalling packages again.".format(self.stack_root_folder)
raise Fail(msg)
def compute_actual_version(self):
"""
After packages are installed, determine what the new actual version is.
"""
# If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
# to correct value if it is not
self.actual_version = None
self.repo_version_with_build_number = None
if self.repository_version:
m = re.search("[\d\.]+-\d+", self.repository_version)
if m:
# Contains a build number
self.repo_version_with_build_number = self.repository_version
self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far.
self.put_structured_out(self.structured_output)
Logger.info("Attempting to determine actual version with build number.")
Logger.info("Old versions: {0}".format(self.old_versions))
new_versions = get_hdp_versions(self.stack_root_folder)
Logger.info("New versions: {0}".format(new_versions))
deltas = set(new_versions) - set(self.old_versions)
Logger.info("Deltas: {0}".format(deltas))
# Get version without build number
normalized_repo_version = self.repository_version.split('-')[0]
if 1 == len(deltas):
self.actual_version = next(iter(deltas)).strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
write_actual_version_to_history_file(normalized_repo_version, self.actual_version)
Logger.info(
"Found actual version {0} by checking the delta between versions before and after installing packages".format(
self.actual_version))
else:
# If the first install attempt does a partial install and is unable to report this to the server,
# then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
Logger.info("Cannot determine actual version installed by checking the delta between versions "
"before and after installing package")
Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed")
self.actual_version = self.find_best_fit_version(new_versions, self.repository_version)
if self.actual_version is not None:
self.actual_version = self.actual_version.strip()
self.structured_output['actual_version'] = self.actual_version
self.put_structured_out(self.structured_output)
Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version))
else:
# It's likely that this host does not have any Stack Components installed, so only contains AMS.
# So just use repo version value provided by server (we already put it to structured output)
if not os.path.exists(self.stack_root_folder):
# Special case when this host does not contain any HDP components, but still contains other components like AMS.
msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \
"Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder)
Logger.info(msg)
else:
msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \
"Try reinstalling packages again.".format(self.stack_root_folder)
raise Fail(msg)
