def check_partial_install(self): """ If an installation did not complete successfully, check if installation was partially complete and log the partially completed version to REPO_VERSION_HISTORY_FILE. :return: """ Logger.info( "Installation of packages failed. Checking if installation was partially complete" ) Logger.info("Old versions: {0}".format(self.old_versions)) new_versions = get_stack_versions(self.stack_root_folder) Logger.info("New versions: {0}".format(new_versions)) deltas = set(new_versions) - set(self.old_versions) Logger.info("Deltas: {0}".format(deltas)) # Get version without build number normalized_repo_version = self.repository_version.split('-')[0] if 1 == len(deltas): # Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE partial_install_version = next(iter(deltas)).strip() write_actual_version_to_history_file(normalized_repo_version, partial_install_version) Logger.info("Version {0} was partially installed. ".format( partial_install_version))
def compute_actual_version(self): """ After packages are installed, determine what the new actual version is. """ # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed # to correct value if it is not self.actual_version = None self.repo_version_with_build_number = None if self.repository_version: m = re.search("[\d\.]+-\d+", self.repository_version) if m: # Contains a build number self.repo_version_with_build_number = self.repository_version self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far. self.put_structured_out(self.structured_output) Logger.info("Attempting to determine actual version with build number.") Logger.info("Old versions: {0}".format(self.old_versions)) new_versions = get_hdp_versions(self.stack_root_folder) Logger.info("New versions: {0}".format(new_versions)) deltas = set(new_versions) - set(self.old_versions) Logger.info("Deltas: {0}".format(deltas)) # Get version without build number normalized_repo_version = self.repository_version.split('-')[0] if 1 == len(deltas): self.actual_version = next(iter(deltas)).strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) write_actual_version_to_history_file(normalized_repo_version, self.actual_version) Logger.info( "Found actual version {0} by checking the delta between versions before and after installing packages".format( self.actual_version)) else: # If the first install attempt does a partial install and is unable to report this to the server, # then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version Logger.info("Cannot determine actual version installed by checking the delta between versions " "before and after installing package") Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed") self.actual_version = self.find_best_fit_version(new_versions, self.repository_version) if self.actual_version is not None: self.actual_version = self.actual_version.strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version)) else: msg = "Could not determine actual version installed. Try reinstalling packages again." raise Fail(msg)
def compute_actual_version(self): """ After packages are installed, determine what the new actual version is. """ # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed # to correct value if it is not self.actual_version = None self.repo_version_with_build_number = None if self.repository_version: m = re.search("[\d\.]+-\d+", self.repository_version) if m: # Contains a build number self.repo_version_with_build_number = self.repository_version self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far. self.put_structured_out(self.structured_output) Logger.info("Attempting to determine actual version with build number.") Logger.info("Old versions: {0}".format(self.old_versions)) new_versions = get_stack_versions(self.stack_root_folder) Logger.info("New versions: {0}".format(new_versions)) deltas = set(new_versions) - set(self.old_versions) Logger.info("Deltas: {0}".format(deltas)) # Get version without build number normalized_repo_version = self.repository_version.split('-')[0] if 1 == len(deltas): self.actual_version = next(iter(deltas)).strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) write_actual_version_to_history_file(normalized_repo_version, self.actual_version) Logger.info( "Found actual version {0} by checking the delta between versions before and after installing packages".format( self.actual_version)) else: # If the first install attempt does a partial install and is unable to report this to the server, # then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version Logger.info("Cannot determine actual version installed by checking the delta between versions " "before and after installing package") Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed") self.actual_version = self.find_best_fit_version(new_versions, self.repository_version) if self.actual_version is not None: self.actual_version = self.actual_version.strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version)) else: msg = "Could not determine actual version installed. Try reinstalling packages again." raise Fail(msg)
def compute_actual_version(self): """ After packages are installed, determine what the new actual version is, in order to save it. """ Logger.info( "Attempting to determine actual version with build number.") Logger.info("Old versions: {0}".format(self.old_versions)) new_versions = get_hdp_versions() Logger.info("New versions: {0}".format(new_versions)) deltas = set(new_versions) - set(self.old_versions) Logger.info("Deltas: {0}".format(deltas)) # Get HDP version without build number normalized_repo_version = self.repository_version.split('-')[0] if 1 == len(deltas): self.actual_version = next(iter(deltas)).strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) write_actual_version_to_history_file(normalized_repo_version, self.actual_version) else: Logger.info( "Cannot determine a new actual version installed by using the delta method." ) # If the first install attempt does a partial install and is unable to report this to the server, # then a subsequent attempt will report an empty delta. For this reason, it is important to search the # repo version history file to determine if we previously did write an actual_version. self.actual_version = read_actual_version_from_history_file( normalized_repo_version) if self.actual_version is not None: self.actual_version = self.actual_version.strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) Logger.info( "Found actual version {0} by parsing file {1}".format( self.actual_version, REPO_VERSION_HISTORY_FILE)) elif self.repo_version_with_build_number is None: # It's likely that this host does not have any Stack Components installed, so only contains AMS. # So just use repo version value provided by server (we already put it to structured output) if not os.path.exists(self.stack_root_folder): # Special case when this host does not contain any HDP components, but still contains other components like AMS. msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \ "Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder) Logger.info(msg) else: msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \ "Try reinstalling packages again.".format(self.stack_root_folder) raise Fail(msg)
def compute_actual_version(self): """ After packages are installed, determine what the new actual version is, in order to save it. """ Logger.info( "Attempting to determine actual version with build number.") Logger.info("Old versions: {0}".format(self.old_versions)) new_versions = get_hdp_versions() Logger.info("New versions: {0}".format(new_versions)) deltas = set(new_versions) - set(self.old_versions) Logger.info("Deltas: {0}".format(deltas)) # Get HDP version without build number normalized_repo_version = self.repository_version.split('-')[0] if 1 == len(deltas): self.actual_version = next(iter(deltas)).strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) write_actual_version_to_history_file(normalized_repo_version, self.actual_version) else: Logger.info( "Cannot determine a new actual version installed by using the delta method." ) # If the first install attempt does a partial install and is unable to report this to the server, # then a subsequent attempt will report an empty delta. For this reason, it is important to search the # repo version history file to determine if we previously did write an actual_version. self.actual_version = read_actual_version_from_history_file( normalized_repo_version) if self.actual_version is not None: self.actual_version = self.actual_version.strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) Logger.info( "Found actual version {0} by parsing file {1}".format( self.actual_version, REPO_VERSION_HISTORY_FILE)) elif self.repo_version_with_build_number is None: msg = "Could not determine actual version installed. Try reinstalling packages again." raise Fail(msg)
def check_partial_install(self): """ If an installation did not complete successfully, check if installation was partially complete and log the partially completed version to REPO_VERSION_HISTORY_FILE. :return: """ Logger.info("Installation of packages failed. Checking if installation was partially complete") Logger.info("Old versions: {0}".format(self.old_versions)) new_versions = get_hdp_versions(self.stack_root_folder) Logger.info("New versions: {0}".format(new_versions)) deltas = set(new_versions) - set(self.old_versions) Logger.info("Deltas: {0}".format(deltas)) # Get version without build number normalized_repo_version = self.repository_version.split('-')[0] if 1 == len(deltas): # Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE partial_install_version = next(iter(deltas)).strip() write_actual_version_to_history_file(normalized_repo_version, partial_install_version) Logger.info("Version {0} was partially installed. ".format(partial_install_version))
def test_read_and_write_repo_version_history(self): f, filename = tempfile.mkstemp() try: # Check read of empty file repo_version_history.REPO_VERSION_HISTORY_FILE = filename repo_version_history.Logger = logging.getLogger() result = repo_version_history.read_actual_version_from_history_file('2.3.2.0') self.assertEquals(result, None) # Check read of single value repo_version_history.write_actual_version_to_history_file('2.3.2.0', '2.3.2.0-210') result = repo_version_history.read_actual_version_from_history_file('2.3.2.0') self.assertEquals(result, '2.3.2.0-210') # Check read after update repo_version_history.write_actual_version_to_history_file('2.3.2.0', '2.3.2.0-2716') result = repo_version_history.read_actual_version_from_history_file('2.3.2.0') self.assertEquals(result, '2.3.2.0-2716') # Check read after update repo_version_history.write_actual_version_to_history_file('2.3.2.0', '2.3.2.0-2758') result = repo_version_history.read_actual_version_from_history_file('2.3.2.0') self.assertEquals(result, '2.3.2.0-2758') # Check read after writing down version for another stack repo_version_history.write_actual_version_to_history_file('2.3.1.0', '2.3.1.0-27') result = repo_version_history.read_actual_version_from_history_file('2.3.1.0') self.assertEquals(result, '2.3.1.0-27') result = repo_version_history.read_actual_version_from_history_file('2.3.2.0') self.assertEquals(result, '2.3.2.0-2758') # Check read of another stack result = repo_version_history.read_actual_version_from_history_file('2.3.0.0') self.assertEquals(result, None) finally: os.unlink(filename)
def compute_actual_version(self): """ After packages are installed, determine what the new actual version is. """ # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed # to correct value if it is not self.actual_version = None self.repo_version_with_build_number = None if self.repository_version: m = re.search("[\d\.]+-\d+", self.repository_version) if m: # Contains a build number self.repo_version_with_build_number = self.repository_version self.structured_output[ 'actual_version'] = self.repo_version_with_build_number # This is the best value known so far. self.put_structured_out(self.structured_output) Logger.info( "Attempting to determine actual version with build number.") Logger.info("Old versions: {0}".format(self.old_versions)) new_versions = get_hdp_versions(self.stack_root_folder) Logger.info("New versions: {0}".format(new_versions)) deltas = set(new_versions) - set(self.old_versions) Logger.info("Deltas: {0}".format(deltas)) # Get version without build number normalized_repo_version = self.repository_version.split('-')[0] if 1 == len(deltas): self.actual_version = next(iter(deltas)).strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) write_actual_version_to_history_file(normalized_repo_version, self.actual_version) Logger.info( "Found actual version {0} by checking the delta between versions before and after installing packages" .format(self.actual_version)) else: # If the first install attempt does a partial install and is unable to report this to the server, # then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version Logger.info( "Cannot determine actual version installed by checking the delta between versions " "before and after installing package") Logger.info( "Will try to find for the actual version by searching for best possible match in the list of versions installed" ) self.actual_version = self.find_best_fit_version( new_versions, self.repository_version) if self.actual_version is not None: self.actual_version = self.actual_version.strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) Logger.info( "Found actual version {0} by searching for best possible match" .format(self.actual_version)) else: # It's likely that this host does not have any Stack Components installed, so only contains AMS. # So just use repo version value provided by server (we already put it to structured output) if not os.path.exists(self.stack_root_folder): # Special case when this host does not contain any HDP components, but still contains other components like AMS. msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \ "Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder) Logger.info(msg) else: msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \ "Try reinstalling packages again.".format(self.stack_root_folder) raise Fail(msg)
def compute_actual_version(self): """ After packages are installed, determine what the new actual version is. """ # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed # to correct value if it is not self.actual_version = None self.repo_version_with_build_number = None if self.repository_version: m = re.search("[\d\.]+-\d+", self.repository_version) if m: # Contains a build number self.repo_version_with_build_number = self.repository_version self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far. self.put_structured_out(self.structured_output) Logger.info("Attempting to determine actual version with build number.") Logger.info("Old versions: {0}".format(self.old_versions)) new_versions = get_hdp_versions(self.stack_root_folder) Logger.info("New versions: {0}".format(new_versions)) deltas = set(new_versions) - set(self.old_versions) Logger.info("Deltas: {0}".format(deltas)) # Get version without build number normalized_repo_version = self.repository_version.split('-')[0] if 1 == len(deltas): self.actual_version = next(iter(deltas)).strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) write_actual_version_to_history_file(normalized_repo_version, self.actual_version) Logger.info( "Found actual version {0} by checking the delta between versions before and after installing packages".format( self.actual_version)) else: # If the first install attempt does a partial install and is unable to report this to the server, # then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version Logger.info("Cannot determine actual version installed by checking the delta between versions " "before and after installing package") Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed") self.actual_version = self.find_best_fit_version(new_versions, self.repository_version) if self.actual_version is not None: self.actual_version = self.actual_version.strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version)) else: # It's likely that this host does not have any Stack Components installed, so only contains AMS. # So just use repo version value provided by server (we already put it to structured output) if not os.path.exists(self.stack_root_folder): # Special case when this host does not contain any HDP components, but still contains other components like AMS. msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \ "Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder) Logger.info(msg) else: msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \ "Try reinstalling packages again.".format(self.stack_root_folder) raise Fail(msg)