def checkpoint_setup(cp, manifest_server_obj):
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
"""Sets up and defines all checkpoints. All checkpoints must
have an entry (in chronological order) here.
format:
step_setup( message, name, [dataset list])
message: the message to print to the screen to indicate the step
name: ascii name for the step.
dataset list: list with the names of the datasets to snapshot
"""
finalizer_script_list = get_manifest_list(manifest_server_obj,
FINALIZER_SCRIPT_NAME)
# Set up a checkpoint for each finalizer script specified.
for script in finalizer_script_list:
checkpoint_message = get_manifest_value(manifest_server_obj,
FINALIZER_SCRIPT_NAME_TO_CHECKPOINT_MESSAGE % script)
checkpoint_name = get_manifest_value(manifest_server_obj,
FINALIZER_SCRIPT_NAME_TO_CHECKPOINT_NAME % script)
if checkpoint_name is None :
dc_log = logging.getLogger(DC_LOGGER_NAME)
dc_log.error("The checkpoint name to use for the " \
"finalizer script %s is missing." % script)
dc_log.error("Please check your manifest file.")
return -1
if checkpoint_message is None:
checkpoint_message = "Executing " + script
cp.step_setup(checkpoint_message, checkpoint_name,
[cp.get_build_area_dataset() + BUILD_DATA])
return 0
def determine_chckpnt_avail(cp, manifest_server_obj):
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
""" Determine if checkpointing is available by reading
the manifest file variable and checking to see if
zfs is on they system.
"""
dc_log = logging.getLogger(DC_LOGGER_NAME)
# The checkpointing available is set to true by default in the
# checkpointing class __init__ method. Here we check to see if
# checkpointing is disabled in the manifest file. If so, return
if get_manifest_value(manifest_server_obj,
CHECKPOINT_ENABLE).lower() == "false":
cp.set_checkpointing_avail(False)
return
# Check to see if checkpointing is disabled because zfs is not on
# the system.
try:
if os.access('/sbin/zfs', os.X_OK) == True:
return
except OSError:
dc_log.error("Unable to determine if ZFS is available "
"on the system. Checkpointing will be unavailable")
# zfs not on the system so no checkpointing
cp.set_checkpointing_avail(False)
cp.set_zfs_found(False)
return
" boot archive build area, media area, grub setup type.")
# collect input arguments from what this script sees as a commandline.
MFEST_SOCKET = sys.argv[1] # Manifest reader socket
PKG_IMG_PATH = sys.argv[2] # package image area mountpoint
GRUB_SETUP_TYPE = sys.argv[6] # Grub setup type
# get the manifest reader object from the socket
MANIFEST_READER_OBJ = ManifestRead(MFEST_SOCKET)
# if a string is specified in the manifest to be used as the title of the grub
# menu entries, that string will be used. Otherwise, use the first
# line of /etc/release as the title
# Get grub title from manifest, if any
RELEASE = get_manifest_value(MANIFEST_READER_OBJ, GRUB_TITLE)
if RELEASE is not None:
# User specified a special grub menu, record that in .image_info
IMG_INFO_FD = None
try:
IMG_INFO_PATH = PKG_IMG_PATH + "/" + IMAGE_INFO_FILE
try:
IMG_INFO_FD = open(IMG_INFO_PATH, "a+")
IMG_INFO_FD.write(IMAGE_INFO_GRUB_TITLE_KEYWORD + RELEASE + "\n")
except Exception, err:
print >> sys.stderr, sys.argv[0] + \
"Unable to write to " + IMG_INFO_PATH
raise err
finally:
if (IMG_INFO_FD != None):
IMG_INFO_FD.close()
if __name__ == "__main__":
if (len(sys.argv) != 6): # Don't forget sys.argv[0] is the script itself.
raise Exception, (sys.argv[0] + ": Requires 5 args:\n" +
" Reader socket, pkg_image area, temp dir,\n" +
" boot archive build area, media area.")
# collect input arguments from what this script sees as a commandline.
mfest_socket = sys.argv[1] # Manifest reader socket
pkg_img_path = sys.argv[2] # package image area mountpoint
# get the manifest reader object from the socket
manifest_reader_obj = ManifestRead(mfest_socket)
# Get the password for the root user from the manifest
root_passwd_text = get_manifest_value(manifest_reader_obj, ROOT_PASSWD)
is_plaintext = get_manifest_boolean(manifest_reader_obj,
ROOT_PASSWD_PLAINTEXT)
print "root_passwd_text = " + root_passwd_text
print "is_plaintext = " + str(is_plaintext)
if is_plaintext:
encrypted_root_passwd = encrypt_password(root_passwd_text,
alt_root=pkg_img_path)
else:
encrypted_root_passwd = root_passwd_text
print "Encrypted root password: " + encrypted_root_passwd
try:
if (len(sys.argv) != 7): # Don't forget sys.argv[0] is the script itself.
raise Exception, (
sys.argv[0] + ": Requires 6 args:\n" +
" Reader socket, pkg_image area, temp dir,\n" +
" boot archive build area, media area, loader setup type.")
# collect input arguments from what this script sees as a commandline.
MFEST_SOCKET = sys.argv[1] # Manifest reader socket
PKG_IMG_PATH = sys.argv[2] # package image area mountpoint
LOADER_SETUP_TYPE = sys.argv[6] # Grub setup type
# get the manifest reader object from the socket
MANIFEST_READER_OBJ = ManifestRead(MFEST_SOCKET)
# Get default timeout from manifest. if it exists.
TIMEOUT = get_manifest_value(MANIFEST_READER_OBJ, LOADER_DEFAULT_TIMEOUT)
if TIMEOUT is not None and TIMEOUT != DEFAULT_TIMEOUT:
# Open loader.conf file.
try:
LOADER_CONF_FILE = open(PKG_IMG_PATH + "/boot/loader.conf", "a")
except IOError, err:
print >> sys.stderr, "Error opening loader.conf for writing"
raise
LOADER_CONF_FILE.write("autoboot_delay=" + TIMEOUT + "\n")
LOADER_CONF_FILE.close()
CMD = CP + " " + LOADER_MENU_FILES + "/* " + PKG_IMG_PATH + "/boot"
if (LOADER_SETUP_TYPE == "ai"):
# The following entries are the standard "hardwired" entries for AI
def main_func():
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
"""Main processing function for the Distribution Constructor. Will set
up the checkpoints, create the build area, start the socket server to
read the manifest file, create the finalizer script queue and then
start the execution of the finalizer scripts.
"""
dc_log = logging.getLogger(DC_LOGGER_NAME)
# Must be root to run DC. Check for that.
if os.getuid() != 0:
dc_log.error("You must be root to run distro_const")
return 1
# Sets the umask for the DC app
os.umask(022)
cp = dc_ckp.Checkpoints()
try:
# Create the object used to extract the data
manifest_server_obj = get_manifest_server_obj(cp)
except UsageError:
raise
except:
return 1
try:
# Start the socket server
start_manifest_server(manifest_server_obj)
# Set up to shut down socket server cleanly on exit
atexit.register(manifest_server_obj.stop_socket_server)
except ManifestServError:
return 1
# create the build area and determine if
# checkpointing is available. When creating the
# build area we also create the areas for the
# package image (pkg_image), output media (media),
# logfiles (logs) and boot archive (boot_archive).
if ti.create_build_area(cp, manifest_server_obj):
return 1
# Set up the structures for all checkpoints
if cp.get_checkpointing_avail() == True:
if dc_ckp.checkpoint_setup(cp, manifest_server_obj):
return 1
# Parse the command line so we know to resume (and where) or not
ret = parse_command_line(cp, manifest_server_obj)
if ret == 1:
# Don't continue processing. Return but nothing is wrong.
return 0
if ret == -1:
# Error parsing the command line. Return indicating such.
return 1
# The build is going to start, will start logging
(simple_log_name, detail_log_name) = \
dcu.setup_dc_logfile_names(cp.get_build_area_mntpt() + LOGS)
dc_log.info("Simple Log: " + simple_log_name)
dc_log.info("Detail Log: " + detail_log_name)
dcu.add_file_logging(simple_log_name, detail_log_name)
dc_log.info("Build started " + time.asctime(time.localtime()))
dc_log.info("Distribution name: " +
(dcu.get_manifest_value(manifest_server_obj, DISTRO_NAME)))
dataset = cp.get_build_area_dataset()
if dataset is not None:
dc_log.info("Build Area dataset: " + dataset)
dc_log.info("Build Area mount point: " + cp.get_build_area_mntpt())
# If we're doing a build that is a -r or -R build, then
# we don't need to cleanup.
if cp.get_resume_step() == -1:
# Cleanup the pkg_image area via either a remove of the files
# if there is not checkpointing or rolling back to the @empty
# snapshot if there is checkpointing.
if cleanup_build_data_area(cp) != 0:
dc_log.info("Build completed " + time.asctime(time.localtime()))
dc_log.info("Build failed.")
return 1
stop_on_err_bool = dcu.get_manifest_boolean(manifest_server_obj,
STOP_ON_ERR)
# register the scripts with the finalizer
build_area = cp.get_build_area_mntpt()
pkg_img_area = build_area + PKG_IMAGE
tmp_area = build_area + TMP
ba_build_area = build_area + BOOT_ARCHIVE
media_dir = build_area + MEDIA
finalizer_obj = DCFinalizer([manifest_server_obj.get_sockname(),
pkg_img_area, tmp_area, ba_build_area,
media_dir])
if (finalizer_obj.change_exec_params(stop_on_err=stop_on_err_bool,
logger_name = DC_LOGGER_NAME) != 0):
dc_log.error("Unable to set stop on error or logger name "
"for finalizer")
status = dc_ckp.add_finalizer_scripts(cp, manifest_server_obj,
finalizer_obj)
if (status != SUCCESS):
dc_log.info("Build completed " + time.asctime(time.localtime()))
dc_log.info("Build failed.")
return (status)
# Execute the finalizer scripts
status = finalizer_obj.execute()
dc_log.info("Build completed " + time.asctime(time.localtime()))
if (status != 0):
dc_log.info("Build failed.")
else:
dc_log.info("Build is successful.")
return (status)
def parse_command_line(cp, manifest_server_obj):
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
"""Parse the command line options.
usage: dist_const build [-R] manifest-file
dist_const build [-r <integer or string>] manifest-file
dist_const build [-p <integer or string>] manifest-file
dist_const build [-l] manifest-file
-R will resume from the last executed step
-r will resume from the specified step
-p will pause at the specified step
-l will list the valid steps to resume/pause at
Also, verify that the pause step and the resume step are valid. The
pause step must be one of the steps. The resume step must be less than
or equal to the last step executed. This method will also remove the
zfs snapshots and the .step file for steps equal to or that come after
the step where execution will start. For a full build or a pause, this
means all checkpoints will be removed. For a resume, the checkpoint to
be resumed from and all later checkpoints are removed.
Return 0 = success
1 = success but don't continue on with building.
-1 = Error
"""
subcommand = sys.argv[1]
# See if the user has specified a step to resume from in the manifest
# file. If so, keep it around. If the user has also specified one on
# the command line, that will overwrite the manifest value.
stepname = dcu.get_manifest_value(manifest_server_obj,
CHECKPOINT_RESUME)
if stepname is not None:
step = dc_ckp.step_from_name(cp, stepname)
if step is None:
return -1
stepno = step.get_step_num()
if stepno:
cp.set_resume_step(stepno)
# Read the command line arguments and parse them.
try:
opts2 = getopt.getopt(sys.argv[2:], "r:p:hRl?")[0]
except getopt.GetoptError:
usage()
if subcommand == "build":
step_resume = False
resume = False
pause = False
do_list = False
dc_log = logging.getLogger(DC_LOGGER_NAME)
for opt, arg in opts2:
if (opt == "-h") or (opt == "-?"):
usage()
# Since all command line opts have to do with
# checkpointing, check here to see
# if checkpointing is available.
if not cp.get_checkpointing_avail():
dc_log.error("Checkpointing is not available")
dc_log.error("Rerun the build without " + opt)
return -1
if opt == "-r":
# Check to see if -r has already been specified
if step_resume == True:
usage()
# resume from the specified step.
step = dc_ckp.step_from_name(cp, arg)
if step is None:
return -1
stepno = step.get_step_num()
cp.set_resume_step(stepno)
step_resume = True
elif opt == "-p":
# Check to see if -p has already been specified
if pause:
usage()
# pause at the specified step.
step = dc_ckp.step_from_name(cp, arg)
if step is None:
return -1
stepno = step.get_step_num()
cp.set_pause_step(stepno)
pause = True
elif opt == "-R":
# resume from the last executed step.
stepno = dc_ckp.determine_resume_step(cp)
if stepno == -1:
dc_log.error("There are no valid " \
"steps to resume from.")
dc_log.error("Please rerun the build " \
"without the -r or -R options.")
return -1
cp.set_resume_step(stepno)
resume = True
elif opt == "-l":
do_list = True
# If a resume step was specified via -r, -R or the manifest file,
# check to see if it's valid. If not, abort the build.
if cp.get_resume_step() != -1:
err = dc_ckp.verify_resume_step(cp, cp.get_resume_step())
if err != 0 :
return -1
# -R and -r not allowed on the same command line.
if resume and step_resume:
dc_log.error("-R and -r cannot be specified for the "
"same build. ")
usage()
# -l and -R, -r, or -p combination not allowed.
# -l must be the only option.
if do_list and (pause or resume or step_resume):
dc_log.error("-l and -R, -r, or -p cannot be "
"specified for the same build")
usage()
# We've checked for the bad combos.
# If a -l was specified, print out the info.
if do_list:
# query for valid resume/pause steps
# All steps are valid to pause at. The
# steps that are valid to resume from
# will be marked "resumable"
laststep = dc_ckp.determine_resume_step(cp)
dc_log.error("\nStep Resumable Description")
dc_log.error("-------------- --------- -------------")
for step_obj in cp.step_list:
if laststep != -1 \
and step_obj.get_step_num() <= laststep:
r_flag = "X"
else:
r_flag = " "
dc_log.error("%s%s%s" % \
(step_obj.get_step_name().ljust(15),
r_flag.center(10),
step_obj.get_step_message().ljust(10)))
return 1
# If -r/-R and -p were both specified,
# the pause step must be later than the resume.
if cp.get_pause_step() <= cp.get_resume_step() :
dc_log.error("The resume step must be earlier "
"than the pause step.")
usage()
if cp.get_resume_step() != -1 :
dc_ckp.verify_resume_state(cp)
# destroy all zfs snapshots and .step files for
# steps after the resume step
for step_obj in cp.step_list[cp.get_resume_step() + 1:]:
dc_ckp.delete_checkpoint(step_obj)
else :
# Delete all zfs snapshots and .step files
for step_obj in cp.step_list:
dc_ckp.delete_checkpoint(step_obj)
else :
# Currently we only support the build subcommand
dc_log.error("Invalid or missing subcommand")
usage()
return 0
BA_ARCHFILE = PKG_IMG_MNT_PT + BA_FILENAME_AMD64
BA_BUILD = TMP_DIR + "/" + KERNEL_ARCH
STRIP_ARCHIVE = True
else:
BA_ARCHFILE = PKG_IMG_MNT_PT + BA_FILENAME_ALL
BA_BUILD = BA_MASTER
STRIP_ARCHIVE = False
# Location of the lofi file mountpoint, known only to this file.
BA_LOFI_MNT_PT = TMP_DIR + "/ba_lofimnt"
# get the manifest reader object from the socket
MANIFEST_READER_OBJ = ManifestRead(MFEST_SOCKET)
# Boot archive compression type and level, and padding amount.
BA_COMPR_LEVEL = get_manifest_value(MANIFEST_READER_OBJ,
BOOT_ARCHIVE_COMPRESSION_LEVEL)
# compression level is not applicable to Sparc platform
if not IS_SPARC and BA_COMPR_LEVEL is None:
raise Exception, (sys.argv[0] +
": boot archive compression level missing from manifest")
BA_COMPR_TYPE = get_manifest_value(MANIFEST_READER_OBJ,
BOOT_ARCHIVE_COMPRESSION_TYPE)
if BA_COMPR_TYPE is None:
raise Exception, (sys.argv[0] +
": boot archive compression type missing from manifest")
PADDING = -1
BA_PAD_SIZE_STR = get_manifest_value(MANIFEST_READER_OBJ,
BOOT_ARCHIVE_SIZE_PAD)
def create_build_area(ckp, manifest_server_obj):
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
"""Create the build area. This may be a normal directory or
a zfs dataset. If it is specified and doesn't start with / a
zfs dataset will attempt to be created. If it starts with a /,
check to see if it's a zfs mountpoint. If not, a normal
directory (mkdir) will be created. This is where all the sub
working directories are created. This includes the pkg_image, logs,
media and boot_archive directories that reside under the build area.
Args:
ckp - checkpointing object
manifest_server_obj: manifest data extraction object
Returns:
-1 on failure
0 on success
"""
# Check manifest file and existence of zfs to see if
# checkpointing is possible.
dc_ckp.determine_chckpnt_avail(ckp, manifest_server_obj)
dc_log = logging.getLogger(DC_LOGGER_NAME)
# Read the build_area from the manifest file. This can be either
# a zfs dataset or a mountpoint.
build_area = dcu.get_manifest_value(manifest_server_obj, BUILD_AREA)
if build_area is None:
dc_log.error(BUILD_AREA + " in the manifest file is invalid " \
"or missing. Build aborted")
return -1
# First check build_area to see
# if there is a leading /. If there isn't, it has to be a zfs dataset.
if build_area.startswith('/'):
# Leading /. build_area can be either a zfs mountpoint or
# a ufs mountpoint.
if ckp.get_zfs_found():
# zfs is on the system so it's at least possible
# that build_area is a zfs mountpoint.
# zfs list -o mountpoint <build_area> will return
# the mountpoint for the build_area specified.
cmd = "/usr/sbin/zfs list -H -o \"mountpoint\" " \
+ build_area
try:
mntpt = Popen(cmd,
shell=True,
universal_newlines=True,
stdout=PIPE).communicate()[0].strip()
except OSError:
dc_log.error("Error determining if the build " \
"area exists")
return -1
# zfs list -H -o name <build_area> will return
# the zfs dataset associated with build_area
cmd = "/usr/sbin/zfs list -H -o \"name\" " \
+ build_area
try:
dataset = Popen(cmd,
shell=True,
universal_newlines=True,
stdout=PIPE).communicate()[0].strip()
except OSError:
dc_log.error("Error finding the build area dataset")
return -1
# If we have found a dataset, check to see if
# the mountpoint and the build_area are the same.
# If so, the build_area that was specified is
# a zfs mountpoint that directly correlates to
# a zfs dataset. If the mountpoint and the build_area
# are not the same, there is not a direct match
# and we can't use checkpointing.
if dataset:
if mntpt == build_area:
# We have a zfs dataset and
# mountpoint so we don't have
# to create one. Just save
# the dataset and mountpoint ofr
# later use and create the subdirs.
ckp.set_build_area_dataset(dataset)
ckp.set_build_area_mntpt(mntpt)
# And now create the subdirs
# for pkg_image, media, logs,
# tmp, and boot_archive
ret = create_subdirs(ckp)
if ret:
return -1
return 0
# We have a build area that doesn't
# have a direct matchup to a mountpoint.
# Checkpointing must not be used. If
# we checkpoint, we run the risk of
# rollinging back more data than would
# be wise. ex. build area is
# /export/home/someone but the mntpt
# is /export/home.
ckp.set_checkpointing_avail(False)
ckp.set_build_area_mntpt(build_area)
# And now create the subdirs
# for pkg_image, media, logs,
# tmp and boot_archive
ret = create_subdirs(ckp)
if ret:
return -1
return 0
# No zfs on the system or no zfs dataset that relates
# to the build_area, create a ufs style dir.
ckp.set_checkpointing_avail(False)
ret = create_ufs_dir(build_area)
if ret:
dc_log.error("Unable to create the build area at " + build_area)
return -1
else:
ckp.set_build_area_mntpt(build_area)
# And now create the subdirs
# for pkg_image, media, logs,
# tmp and boot_archive
ret = create_subdirs(ckp)
if ret:
return -1
else:
# build_area doesn't start with a /, thus it
# has to be a zfs dataset.
# Check to see if zfs is on the system. If
# not we have an error since a zfs dataset
# was specified.
if not ckp.get_zfs_found():
dc_log.error("ZFS dataset was specified for the build area,")
dc_log.error("but zfs is not installed on the system.")
return -1
# create zfs fs
ret = create_zfs_fs(build_area)
if ret:
return -1
# The zfs fs was created, get the associated mountpoint
cmd = "/usr/sbin/zfs list -H -o \"mountpoint\" " + build_area
try:
mntpt = Popen(cmd,
shell=True,
universal_newlines=True,
stdout=PIPE).communicate()[0].strip()
except OSError:
dc_log.error("Unable to get the mountpoint for the "
"zfs dataset " + build_area)
return -1
ckp.set_build_area_mntpt(mntpt)
ckp.set_build_area_dataset(build_area)
# And now create the subdirs
# for pkg_image, media, logs,
# tmp and boot_archive
ret = create_subdirs(ckp)
if ret:
return -1
return 0
raise Exception(sys.argv[0] + ": Requires 5 args:\n" +
" Reader socket, pkg_image area, tmp dir,\n" +
" bootroot build area, media area")
# Collect input arguments from what
# this script sees as a commandline.
MFEST_SOCKET = sys.argv[1] # Manifest reader socket
PKG_IMG_MNT_PT = sys.argv[2] # package image area mountpoint
TMP_DIR = sys.argv[3] # temp directory to contain boot archive
UNSET_AUTH_LIST = []
UNSET_MIRROR_LIST = []
MANIFEST_SERVER_OBJ = ManifestRead(MFEST_SOCKET)
PKG_URL = dcu.get_manifest_value(MANIFEST_SERVER_OBJ, DEFAULT_MAIN_URL)
PKG_AUTH = dcu.get_manifest_value(MANIFEST_SERVER_OBJ,
DEFAULT_MAIN_AUTHNAME)
if PKG_AUTH is None:
raise Exception(sys.argv[0] +
": pkg publisher is missing from manifest")
if PKG_URL is None:
raise Exception(sys.argv[0] + ": pkg url is missing from manifest")
QUIT_ON_PKG_FAILURE = dcu.get_manifest_value(MANIFEST_SERVER_OBJ,
STOP_ON_ERR).lower()
# Initialize log instance
# Once initialized, STDOUT messages will show up in the detailed
# log and STDERR messages will show up in the detailed and simple
