def create_tarball(self): """This method creates a tar file based on the release manifest. Returns: (str): returns the tar file """ tarball_path = ybutils.get_release_file(self.repository, self.release_name, self.build_type) ybutils.log_message(logging.INFO, "Exporting release tarball") # TODO(bogdan,ram): hack around yugabyte untaring into a top-level dir named by repo. # This should only be the repo name passed in from the respective yb_release.py in either # yugabyte or devops. prefix_dir = "" if self.release_name == "yugabyte": prefix_dir = "{}-{}".format( self.release_name, ybutils.get_default_release_version(self.repository)) with tarfile.open(tarball_path, "w:gz") as tar: for folder_key in self.release_manifest: if folder_key.startswith("/"): raise YBOpsRuntimeError("Manifest file keys should not start with /") for file_pattern in self.release_manifest[folder_key]: # If the file pattern starts with /, we assume that the pattern includes # absolute path, if not it is relative to the repository path. if not file_pattern.startswith("/"): file_pattern = os.path.join(self.repository, file_pattern) for file_path in glob.glob(file_pattern): path_within_tarball = os.path.join( prefix_dir, folder_key, os.path.basename(file_path)) tar.add(file_path, arcname=path_within_tarball) ybutils.log_message(logging.INFO, "Generated tarball: {}".format(tarball_path)) return tarball_path
def check_for_uncommitted_changes(self): """This method checks if the git repository isn't dirty. """ local_changes = subprocess.check_output( ["git", "diff", "origin/master"]).decode("utf-8").strip() if not local_changes: return ybutils.log_message(logging.ERROR, "Repository {} appears to have uncommitted changes. " "The source release will not include your local changes." .format(self.repository)) if not self.force_yes: ybutils.confirm_prompt("Continue?")
def check_for_local_commits(self): """This method checks if there is local commits which haven't been pushed upstream. """ subprocess.call(["git", "fetch", "origin"]) local_commits = subprocess.check_output( ["git", "log", "origin/master..HEAD", "--oneline"]) if not local_commits: return ybutils.log_message( logging.ERROR, "Repository {} appears to have local commits:\n {} " "This should not be an official release!".format( self.repository, local_commits)) if not self.force_yes: ybutils.confirm_prompt("Continue?")
def generate_release(self): """This method check if we have local uncommitted changes, or local commits and confirms with the user, if accepted, we proceed to create a tar based on the release manifest and write the checksum files as well. Returns: (str): returns the generated tar file name """ tar_file = None current_dir = os.getcwd() try: os.chdir(self.repository) self.check_for_uncommitted_changes() self.check_for_local_commits() tar_file = self.create_tarball() ybutils.log_message(logging.INFO, "Release generation succeeded!") finally: os.chdir(current_dir) return tar_file
action='store_true', help= 'Untar release in "target/universal" directory or in --destination if specified.' ) # IMPORTANT: DO NOT REMOVE THIS FLAG. REQUIRED BY INTERNAL INFRA. parser.add_argument('--force', help='Force no user input', action="store_true") args = parser.parse_args() try: init_env(logging.INFO) script_dir = os.path.dirname(os.path.realpath(__file__)) _, err = Popen(["sbt", "clean"], stderr=PIPE).communicate() if err: raise RuntimeError(err) log_message(logging.INFO, "Kick off SBT universal packaging") # Ignore any error output from packaging as npm is expected to have some warnings. Popen(["sbt", "universal:packageZipTarball"], stderr=PIPE).communicate() packaged_files = glob.glob( os.path.join(script_dir, "target", "universal", "yugaware*.tgz")) if args.unarchived: package_tar = packaged_files[0] tar = tarfile.open(package_tar) tar.extractall( args.destination) if args.destination else tar.extractall() else: log_message(logging.INFO, "Get a release file name based on the current commit sha") release_file = get_release_file(script_dir, "yugaware") if len(packaged_files) == 0:
def main(): parser = argparse.ArgumentParser() parser.add_argument('--build', help='Build type (debug/release)', default="release", dest='build_type') parser.add_argument('--build-args', help='Additional arguments to pass to the build script', dest='build_args', default='') parser.add_argument('--publish', action='store_true', help='Publish release to S3.') parser.add_argument('--destination', help='Copy release to Destination folder.') parser.add_argument('--verbose', help='Show verbose output', action='store_true') parser.add_argument('--force', help='Skip prompts', action='store_true') parser.add_argument('--edition', help='Which edition the code is built as.', default=RELEASE_EDITION_ENTERPRISE, choices=RELEASE_EDITION_ALLOWED_VALUES) args = parser.parse_args() init_env(logging.DEBUG if args.verbose else logging.INFO) repository_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) log_message(logging.INFO, "Building Yugabyte code: '{}' build".format(args.build_type)) tmp_dir = tempfile.mkdtemp(suffix=os.path.basename(__file__)) atexit.register(lambda: shutil.rmtree(tmp_dir)) build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml') yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution') os.chdir(repository_root) build_cmd_line = "./yb_build.sh {} --with-assembly --write-build-descriptor {} {}".format( args.build_type, build_desc_path, args.build_args).strip() log_message(logging.INFO, "Build command line: {}".format(build_cmd_line)) if call(build_cmd_line, shell=True) != 0: raise RuntimeError('Build failed') if not os.path.exists(build_desc_path): raise IOError("The build script failed to generate build descriptor file at '{}'".format( build_desc_path)) with open(build_desc_path) as build_desc_file: build_desc = yaml.load(build_desc_file) log_message(logging.INFO, "Build descriptor: {}".format(build_desc)) build_dir = build_desc['build_root'] release_manager = ReleaseManager({"repository": repository_root, "name": "yugabyte", "type": args.build_type, "edition": args.edition, "force_yes": args.force}) # This points to the release manifest within the release_manager, and we are modifying that # directly. release_manifest = release_manager.release_manifest library_packager = LibraryPackager( build_dir=build_dir, seed_executable_patterns=release_manifest['bin'], dest_dir=yb_distribution_dir) library_packager.package_binaries() for release_subdir in ['bin']: if release_subdir in release_manifest: del release_manifest[release_subdir] for root, dirs, files in os.walk(yb_distribution_dir): release_manifest.setdefault(os.path.relpath(root, yb_distribution_dir), []).extend([ os.path.join(root, f) for f in files]) log_message(logging.INFO, "Generating release") # We've already updated the release manifest inside release_manager with the auto-generated # set of executables and libraries to package. release_file = release_manager.generate_release() if args.publish: log_message(logging.INFO, "Publishing release") release_manager.publish_release() elif args.destination: if not os.path.exists(args.destination): raise YBOpsRuntimeError("Destination {} not a directory.".format(args.destination)) shutil.copy(release_file, args.destination)
action='store_true', help= 'Untar release in "target/universal" directory or in --destination if specified.' ) # IMPORTANT: DO NOT REMOVE THIS FLAG. REQUIRED BY INTERNAL INFRA. parser.add_argument('--force', help='Force no user input', action="store_true") args = parser.parse_args() try: init_env(logging.INFO) script_dir = os.path.dirname(os.path.realpath(__file__)) _, err = Popen(["sbt", "clean", "-batch"], stderr=PIPE).communicate() if err: raise RuntimeError(err) log_message(logging.INFO, "Kick off SBT universal packaging") # Ignore any error output from packaging as npm is expected to have some warnings. os.system("df -h; sbt universal:packageZipTarball -batch") log_message(logging.INFO, "Finished running SBT universal packaging.") packaged_files = glob.glob( os.path.join(script_dir, "target", "universal", "yugaware*.tgz")) if args.unarchived: package_tar = packaged_files[0] tar = tarfile.open(package_tar) tar.extractall( args.destination) if args.destination else tar.extractall() else: log_message(logging.INFO, "Get a release file name based on the current commit sha") release_file = get_release_file(script_dir, "yugaware")
import shutil from ybops.utils import init_env, log_message, get_release_file from ybops.common.exceptions import YBOpsRuntimeError """This script packages Anywhere's support packages according to release info. - Rename the package file to have the commit sha in it - Move the package file to the required destination - A higher level user, such as itest, will upload all release packages to s3 release bucket """ parser = argparse.ArgumentParser() parser.add_argument('--destination', help='Copy release to Destination folder.') parser.add_argument('--package', help='Package to rename and copy to Destination') args = parser.parse_args() try: init_env(logging.INFO) script_dir = os.path.dirname(os.path.realpath(__file__)) release_file = get_release_file(script_dir, "yugabundle_support") shutil.copyfile(args.package, release_file) if args.destination: if not os.path.exists(args.destination): raise YBOpsRuntimeError("Destination {} not a directory.".format( args.destination)) shutil.copy(release_file, args.destination) except (OSError, shutil.SameFileError) as e: log_message(logging.ERROR, e) raise e