def compare_builds(self, build, reference='golden'): """ Compares a build against a reference build, (by default named 'golden' build) :param buildname: :param golden: :return: """ compare_build = build + "_vs_" + reference self.logger.info('Starting compare build: %s', compare_build) # Compare all the tag images in the build path against the golden set build_paths = output.get_all_paths( buildname=build, basedir=OUTPUT_BASEDIR) compare_errors = {} for bp in build_paths: build_pb = output.create_from_path(bp, basepath=OUTPUT_BASEDIR) # The reference build is included at the root level of tagtester ref_pb = build_pb.clone( build=reference, basepath=os.path.dirname(__file__)) assert os.path.exists( ref_pb.buildpath), 'Reference build not found!' # We can compare iff both paths exists build_image = build_pb.tagimage ref_image = ref_pb.tagimage if not os.path.exists(build_image): self.logger.warn( 'SKIPPING compare: path does not exist at {}'.format( build_pb.path)) continue if not os.path.exists(ref_image): self.logger.warn( 'SKIPPING compare: path does not exist at {}'.format( ref_pb.path)) # TODO: In this case we want to have the option of making new # ref continue result = image.compare(build_pb.tagimage, ref_pb.tagimage) self.logger.debug('Result for %s: %s', compare_build, result) if result > settings.ImageErrorLevel.SLIGHT: compare_errors[build_image] = result self.logger.warn('Invalid compare result for %s', build_image) self.logger.info( 'Finished compare build %s for %s images! Found %s errors:\n%s', compare_build, len(build_paths), len(compare_errors), compare_errors)