def create_tarball(self):
"""This method creates a tar file based on the release manifest.
Returns:
(str): returns the tar file
"""
tarball_path = ybutils.get_release_file(self.repository,
self.release_name,
self.build_type)
ybutils.log_message(logging.INFO, "Exporting release tarball")
# TODO(bogdan,ram): hack around yugabyte untaring into a top-level dir named by repo.
# This should only be the repo name passed in from the respective yb_release.py in either
# yugabyte or devops.
prefix_dir = ""
if self.release_name == "yugabyte":
prefix_dir = "{}-{}".format(
self.release_name, ybutils.get_default_release_version(self.repository))
with tarfile.open(tarball_path, "w:gz") as tar:
for folder_key in self.release_manifest:
if folder_key.startswith("/"):
raise YBOpsRuntimeError("Manifest file keys should not start with /")
for file_pattern in self.release_manifest[folder_key]:
# If the file pattern starts with /, we assume that the pattern includes
# absolute path, if not it is relative to the repository path.
if not file_pattern.startswith("/"):
file_pattern = os.path.join(self.repository, file_pattern)
for file_path in glob.glob(file_pattern):
path_within_tarball = os.path.join(
prefix_dir, folder_key, os.path.basename(file_path))
tar.add(file_path, arcname=path_within_tarball)
ybutils.log_message(logging.INFO, "Generated tarball: {}".format(tarball_path))
return tarball_path
def check_for_uncommitted_changes(self):
"""This method checks if the git repository isn't dirty.
"""
local_changes = subprocess.check_output(
["git", "diff", "origin/master"]).decode("utf-8").strip()
if not local_changes:
return
ybutils.log_message(logging.ERROR,
"Repository {} appears to have uncommitted changes. "
"The source release will not include your local changes."
.format(self.repository))
if not self.force_yes:
ybutils.confirm_prompt("Continue?")
def check_for_local_commits(self):
"""This method checks if there is local commits which haven't been pushed upstream.
"""
subprocess.call(["git", "fetch", "origin"])
local_commits = subprocess.check_output(
["git", "log", "origin/master..HEAD", "--oneline"])
if not local_commits:
return
ybutils.log_message(
logging.ERROR, "Repository {} appears to have local commits:\n {} "
"This should not be an official release!".format(
self.repository, local_commits))
if not self.force_yes:
ybutils.confirm_prompt("Continue?")
def generate_release(self):
"""This method check if we have local uncommitted changes, or local commits
and confirms with the user, if accepted, we proceed to create a tar based on the
release manifest and write the checksum files as well.
Returns:
(str): returns the generated tar file name
"""
tar_file = None
current_dir = os.getcwd()
try:
os.chdir(self.repository)
self.check_for_uncommitted_changes()
self.check_for_local_commits()
tar_file = self.create_tarball()
ybutils.log_message(logging.INFO, "Release generation succeeded!")
finally:
os.chdir(current_dir)
return tar_file
action='store_true',
help=
'Untar release in "target/universal" directory or in --destination if specified.'
)
# IMPORTANT: DO NOT REMOVE THIS FLAG. REQUIRED BY INTERNAL INFRA.
parser.add_argument('--force', help='Force no user input', action="store_true")
args = parser.parse_args()
try:
init_env(logging.INFO)
script_dir = os.path.dirname(os.path.realpath(__file__))
_, err = Popen(["sbt", "clean"], stderr=PIPE).communicate()
if err:
raise RuntimeError(err)
log_message(logging.INFO, "Kick off SBT universal packaging")
# Ignore any error output from packaging as npm is expected to have some warnings.
Popen(["sbt", "universal:packageZipTarball"], stderr=PIPE).communicate()
packaged_files = glob.glob(
os.path.join(script_dir, "target", "universal", "yugaware*.tgz"))
if args.unarchived:
package_tar = packaged_files[0]
tar = tarfile.open(package_tar)
tar.extractall(
args.destination) if args.destination else tar.extractall()
else:
log_message(logging.INFO,
"Get a release file name based on the current commit sha")
release_file = get_release_file(script_dir, "yugaware")
if len(packaged_files) == 0:
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--build', help='Build type (debug/release)',
default="release",
dest='build_type')
parser.add_argument('--build-args',
help='Additional arguments to pass to the build script',
dest='build_args',
default='')
parser.add_argument('--publish', action='store_true', help='Publish release to S3.')
parser.add_argument('--destination', help='Copy release to Destination folder.')
parser.add_argument('--verbose', help='Show verbose output', action='store_true')
parser.add_argument('--force', help='Skip prompts', action='store_true')
parser.add_argument('--edition', help='Which edition the code is built as.',
default=RELEASE_EDITION_ENTERPRISE,
choices=RELEASE_EDITION_ALLOWED_VALUES)
args = parser.parse_args()
init_env(logging.DEBUG if args.verbose else logging.INFO)
repository_root = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))
log_message(logging.INFO, "Building Yugabyte code: '{}' build".format(args.build_type))
tmp_dir = tempfile.mkdtemp(suffix=os.path.basename(__file__))
atexit.register(lambda: shutil.rmtree(tmp_dir))
build_desc_path = os.path.join(tmp_dir, 'build_descriptor.yaml')
yb_distribution_dir = os.path.join(tmp_dir, 'yb_distribution')
os.chdir(repository_root)
build_cmd_line = "./yb_build.sh {} --with-assembly --write-build-descriptor {} {}".format(
args.build_type, build_desc_path, args.build_args).strip()
log_message(logging.INFO, "Build command line: {}".format(build_cmd_line))
if call(build_cmd_line, shell=True) != 0:
raise RuntimeError('Build failed')
if not os.path.exists(build_desc_path):
raise IOError("The build script failed to generate build descriptor file at '{}'".format(
build_desc_path))
with open(build_desc_path) as build_desc_file:
build_desc = yaml.load(build_desc_file)
log_message(logging.INFO, "Build descriptor: {}".format(build_desc))
build_dir = build_desc['build_root']
release_manager = ReleaseManager({"repository": repository_root,
"name": "yugabyte",
"type": args.build_type,
"edition": args.edition,
"force_yes": args.force})
# This points to the release manifest within the release_manager, and we are modifying that
# directly.
release_manifest = release_manager.release_manifest
library_packager = LibraryPackager(
build_dir=build_dir,
seed_executable_patterns=release_manifest['bin'],
dest_dir=yb_distribution_dir)
library_packager.package_binaries()
for release_subdir in ['bin']:
if release_subdir in release_manifest:
del release_manifest[release_subdir]
for root, dirs, files in os.walk(yb_distribution_dir):
release_manifest.setdefault(os.path.relpath(root, yb_distribution_dir), []).extend([
os.path.join(root, f) for f in files])
log_message(logging.INFO, "Generating release")
# We've already updated the release manifest inside release_manager with the auto-generated
# set of executables and libraries to package.
release_file = release_manager.generate_release()
if args.publish:
log_message(logging.INFO, "Publishing release")
release_manager.publish_release()
elif args.destination:
if not os.path.exists(args.destination):
raise YBOpsRuntimeError("Destination {} not a directory.".format(args.destination))
shutil.copy(release_file, args.destination)
action='store_true',
help=
'Untar release in "target/universal" directory or in --destination if specified.'
)
# IMPORTANT: DO NOT REMOVE THIS FLAG. REQUIRED BY INTERNAL INFRA.
parser.add_argument('--force', help='Force no user input', action="store_true")
args = parser.parse_args()
try:
init_env(logging.INFO)
script_dir = os.path.dirname(os.path.realpath(__file__))
_, err = Popen(["sbt", "clean", "-batch"], stderr=PIPE).communicate()
if err:
raise RuntimeError(err)
log_message(logging.INFO, "Kick off SBT universal packaging")
# Ignore any error output from packaging as npm is expected to have some warnings.
os.system("df -h; sbt universal:packageZipTarball -batch")
log_message(logging.INFO, "Finished running SBT universal packaging.")
packaged_files = glob.glob(
os.path.join(script_dir, "target", "universal", "yugaware*.tgz"))
if args.unarchived:
package_tar = packaged_files[0]
tar = tarfile.open(package_tar)
tar.extractall(
args.destination) if args.destination else tar.extractall()
else:
log_message(logging.INFO,
"Get a release file name based on the current commit sha")
release_file = get_release_file(script_dir, "yugaware")
import shutil
from ybops.utils import init_env, log_message, get_release_file
from ybops.common.exceptions import YBOpsRuntimeError
"""This script packages Anywhere's support packages according to release info.
- Rename the package file to have the commit sha in it
- Move the package file to the required destination
- A higher level user, such as itest, will upload all release packages to s3 release bucket
"""
parser = argparse.ArgumentParser()
parser.add_argument('--destination',
help='Copy release to Destination folder.')
parser.add_argument('--package',
help='Package to rename and copy to Destination')
args = parser.parse_args()
try:
init_env(logging.INFO)
script_dir = os.path.dirname(os.path.realpath(__file__))
release_file = get_release_file(script_dir, "yugabundle_support")
shutil.copyfile(args.package, release_file)
if args.destination:
if not os.path.exists(args.destination):
raise YBOpsRuntimeError("Destination {} not a directory.".format(
args.destination))
shutil.copy(release_file, args.destination)
except (OSError, shutil.SameFileError) as e:
log_message(logging.ERROR, e)
raise e
