def installworld(destdir, worldlog, distriblog, conf="build"):
info('Installing world in {0}', destdir)
info('Log file: {0}', worldlog)
makeconf = objdir("make-${conf}.conf")
sh(
"env MAKEOBJDIRPREFIX=${OBJDIR}",
"make",
"-C ${TRUEOS_ROOT}",
"installworld",
"DESTDIR=${destdir}",
"__MAKE_CONF=${makeconf}",
log=worldlog
)
info('Creating distribution in {0}', destdir)
info('Log file: {0}', distriblog)
sh(
"env MAKEOBJDIRPREFIX=${OBJDIR}",
"make",
"-C ${TRUEOS_ROOT}",
"distribution",
"DESTDIR=${destdir}",
"__MAKE_CONF=${makeconf}",
log=distriblog
)
def make_iso_image():
setfile('${ISO_DESTDIR}/boot/loader.conf', template('${BUILD_CONFIG}/templates/cdrom/loader.conf'))
setfile('${ISO_DESTDIR}/boot/grub/grub.cfg', template('${BUILD_CONFIG}/templates/cdrom/grub.cfg'))
setfile('${ISO_DESTDIR}/.mount.conf', template('${BUILD_CONFIG}/templates/cdrom/mount.conf'))
sh('cp ${WORLD_DESTDIR}/boot/device.hints ${ISO_DESTDIR}/boot/device.hints')
sh('grub-mkrescue -o ${output} ${ISO_DESTDIR} -- -volid ${CDROM_LABEL}')
sha256(output)
def setup_vm():
global vm_proc, termserv_proc
info('Starting up VM')
sh('bhyveload -m ${RAM_SIZE} -d ${OBJDIR}/test-root.ufs ${VM_NAME}')
vm_proc = sh_spawn(
'bhyve -m ${RAM_SIZE} -A -H -P',
'-s 0:0,hostbridge',
'-s 1:0,virtio-net,${tapdev}',
'-s 2:0,ahci-hd,${OBJDIR}/test-root.ufs',
'-s 3:0,ahci-hd,${OBJDIR}/test-swap.bin',
'-s 31,lpc -l com1,${CONSOLE_MASTER}',
'${VM_NAME}'
)
pid = vm_proc.pid
logfile = objdir(e('logs/bhyve.${pid}.log'))
info('Starting telnet server on port {0}', e('${TELNET_PORT}'))
info('Console log file is {0}', logfile)
termserv_proc = sh_spawn(
'python',
'${BUILD_TOOLS}/terminal-server.py',
'-l ${logfile}',
'-c ${CONSOLE_SLAVE}',
'-p ${TELNET_PORT}'
)
on_abort(shutdown_vm)
def setup_rootfs():
buildkernel(e('${KERNCONF}-DEBUG'), ['mach'], buildkernellog)
installworld('${OBJDIR}/test-root', installworldlog, distributionlog, conf="run")
installkernel(e('${KERNCONF}'), '${OBJDIR}/test-root', installkernellog, modules=['mach'], conf="run")
info('Installing overlay files')
sh('rsync -ah ${TESTS_ROOT}/trueos/overlay/ ${OBJDIR}/test-root')
sh('makefs -M ${IMAGE_SIZE} ${OBJDIR}/test-root.ufs ${OBJDIR}/test-root')
def create_package():
if os.path.exists('/usr/local/bin/pigz'):
sh('tar -C ${DEBUG_ROOT} -cvf - . | /usr/local/bin/pigz -c > ${output}', log='/dev/null')
else:
sh('tar -C ${DEBUG_ROOT} -cvJf ${output} .', log='/dev/null')
sha256(output)
def setup_network():
global tapdev
info('Configuring VM networking')
tapdev = sh_str('ifconfig tap create')
info('Using tap device {0}', tapdev)
sh('ifconfig ${tapdev} inet ${HOST_IP} ${NETMASK} up')
def main():
for i in files_to_save:
sh('mv ${WORLD_DESTDIR}/${i} ${WORLD_DESTDIR}/${i}.bak')
chroot('${WORLD_DESTDIR}', 'pkg delete -y -f gcc\* || true')
for i in files_to_save:
sh('mv ${WORLD_DESTDIR}/${i}.bak ${WORLD_DESTDIR}/${i}')
def install_ports():
pkgs = ' '.join(get_port_names(config.ports))
sh('mount -t devfs devfs ${WORLD_DESTDIR}/dev')
err = chroot('${WORLD_DESTDIR}', 'env ASSUME_ALWAYS_YES=yes pkg install -r local -f ${pkgs}', log=logfile, nofail=True)
sh('umount -f ${WORLD_DESTDIR}/dev')
if not os.path.isdir(e('${WORLD_DESTDIR}/data')) or err != 0:
error('Packages installation failed, see log file {0}', logfile)
def pip_install(pkg, no_index=True):
pkg_dir.chdir()
if no_index:
cmd = "pip --isolated install --no-index --upgrade ./%s" % pkg
else:
cmd = "pip --isolated install --upgrade ./%s" % pkg
sh(cmd)
curdir.chdir()
def checkout(pkg, commit):
d = pkg_dir / pkg
d.chdir()
commit = str(commit)
sh("git checkout %s" % commit)
curdir.chdir()
def build_pkgtools():
info('Building freenas-pkgtools')
info('Log file: {0}', pkgtoolslog)
sh(
"env MAKEOBJDIRPREFIX=${OBJDIR}",
"make -C ${BE_ROOT}/freenas-pkgtools obj all install DESTDIR=${tooldir} PREFIX=/usr/local",
log=pkgtoolslog
)
def prepare_jail():
basepath = e('${POUDRIERE_ROOT}/etc/poudriere.d/jails/${jailname}')
sh('mkdir -p ${basepath}')
setfile(e('${basepath}/method'), 'git')
setfile(e('${basepath}/mnt'), e('${JAIL_DESTDIR}'))
setfile(e('${basepath}/version'), e('${FREEBSD_RELEASE_VERSION}'))
setfile(e('${basepath}/arch'), e('${BUILD_ARCH}'))
sh("jail -U root -c name=${jailname} path=${JAIL_DESTDIR} command=/sbin/ldconfig -m /lib /usr/lib /usr/lib/compat")
def create_poudriere_config():
sh('mkdir -p ${DISTFILES_CACHE}')
setfile('${POUDRIERE_ROOT}/etc/poudriere.conf', template('${BUILD_CONFIG}/templates/poudriere.conf', {
'ports_repo': config['repos'].where(name='ports')['path'],
'ports_branch': config['repos'].where(name='ports')['branch'],
}))
tree = e('${POUDRIERE_ROOT}/etc/poudriere.d/ports/p')
sh('mkdir -p', tree)
setfile(pathjoin(tree, 'mnt'), e('${PORTS_OVERLAY}'))
setfile(pathjoin(tree, 'method'), 'git')
def fetch_binary_packages():
if e('${SKIP_PACKAGES_FETCH}'):
return
for i in config.binary_packages:
_, name = os.path.split(i)
if os.path.exists(e('${WORLD_DESTDIR}/usr/ports/packages/${name}')):
continue
info('Fetching package {0}', name)
sh('fetch ${i} -o ${WORLD_DESTDIR}/usr/ports/packages/')
def main():
user = sh_str('id -un')
if user == 'root':
user = '******'
# sh('ssh ${user}@${DOWNLOAD_HOST} rm -rf ${DOWNLOAD_TARGETDIR}')
# sh('ssh ${user}@${DOWNLOAD_HOST} mkdir -p ${DOWNLOAD_TARGETDIR}')
# sh('scp -pr ${RELEASE_STAGEDIR}/* ${user}@${DOWNLOAD_HOST}:${DOWNLOAD_TARGETDIR}/')
ref_date = 0
rel_dir = ''
dirstring = e('${BE_ROOT}/release/${PRODUCT}')
for x in glob.glob("{0}*".format(dirstring)):
if e('${BUILD_ARCH_SHORT}') not in os.listdir(x):
continue
if os.lstat(x).st_ctime > ref_date:
ref_date = os.lstat(x).st_ctime
rel_dir = x
if not rel_dir:
error('Release not found')
if e('${BUILD_TYPE}').lower() in ["master", "stable"]:
buildtimestamp = os.path.basename(rel_dir).split("-")[-1]
downloadtargetdir = e('${DOWNLOAD_BASEDIR}/${MILESTONE}/${buildtimestamp}')
else:
downloadtargetdir = e('${DOWNLOAD_TARGETDIR}')
sh('ssh ${user}@${DOWNLOAD_HOST} rm -rf ${downloadtargetdir}')
sh('ssh ${user}@${DOWNLOAD_HOST} mkdir -p ${downloadtargetdir}')
sh('scp -pr ${rel_dir}/* ${user}@${DOWNLOAD_HOST}:${downloadtargetdir}/')
info('Synchronizing download server to CDN')
sh('ssh ${user}@${DOWNLOAD_HOST} /usr/local/sbin/rsync-mirror.sh')
def buildworld():
info('Building world from ${{TRUEOS_ROOT}}')
info('Log file: {0}', worldlog)
debug('World make.conf: {0}', makeconfbuild)
sh(
"env -u DEBUG -u MAKEFLAGS MAKEOBJDIRPREFIX=${OBJDIR}",
"make",
"-j {0}".format(makejobs),
"-C ${TRUEOS_ROOT}",
"__MAKE_CONF={0}".format(makeconfbuild),
"NOCLEAN=YES",
"buildworld",
log=worldlog
)
def cleandirs():
info('Cleaning previous build products')
if os.path.isdir(e('${INSTUFS_DESTDIR}')):
sh('chflags -R 0 ${INSTUFS_DESTDIR}')
sh('rm -rf ${INSTUFS_DESTDIR}')
sh('rm -rf ${ISO_DESTDIR}')
sh('mkdir -p ${ISO_DESTDIR} ${INSTUFS_DESTDIR}')
def cleanup_env():
global poudriere_proc
info('Cleaning up poudriere environment...')
if poudriere_proc and poudriere_proc.poll() is None:
try:
poudriere_proc.terminate()
poudriere_proc.wait()
except OSError:
info('Cannot kill poudriere, it has probably already terminated')
info('Unmounting ports overlay...')
sh('rm -rf ${PORTS_OVERLAY}')
for cmd in jailconf.get('link', []):
sh('umount -f', cmd['source'])
def install_mininet():
dist.install("git")
if dist.NAME == "Fedora":
mininet_opts = "-fnp"
dist.install("openvswitch", "openvswitch-devel", "openvswitch-test")
sh("systemctl enable openvswitch")
sh("systemctl start openvswitch")
else:
mininet_opts = "-a"
sh("git clone https://github.com/mininet/mininet.git", cwd=args.output_dir)
sh("git checkout %s" % MininetVersion, cwd=os.path.join(args.output_dir, "mininet"))
sh("mininet/util/install.sh %s -s ." % mininet_opts,
"pip2 -q install mininet/",
cwd=args.output_dir)
def create_manifest(pkgs):
info('Creating package manifests')
date = int(time.time())
train = e('${TRAIN}') or 'FreeNAS'
sh(
"env PYTHONPATH=${tooldir}/usr/local/lib",
"${tooldir}/usr/local/bin/create_manifest",
"-P ${pkgdir}/Packages",
"-S ${sequence}",
"-o ${pkgdir}/${PRODUCT}-${sequence}",
"-R ${PRODUCT}-${VERSION}",
"-T ${train}",
"-t ${date}",
*pkgs
)
sh('ln -sf ${PRODUCT}-${sequence} ${pkgdir}/${PRODUCT}-MANIFEST')
def run(cmd=cmd, error_file=error_file):
# sh('rm simple/*/*.pyc')
status = sh(cmd)
if status == 0:
return status
else:
return parse_error(error_file)
def buildkernel(kconf, modules, log):
modules = ' '.join(modules)
info('Building kernel {0} from {1}', kconf, e('${TRUEOS_ROOT}'))
info('Log file: {0}', log)
debug('Kernel configuration file: {0}', kernconf)
debug('Selected modules: {0}', modules)
sh(
"env -u DEBUG -u MAKEFLAGS MAKEOBJDIRPREFIX=${OBJDIR}",
"make",
"-j {0}".format(makejobs),
"-C ${TRUEOS_ROOT}",
"NO_KERNELCLEAN=YES",
"KERNCONF={0}".format(kconf),
"__MAKE_CONF={0}".format(makeconfbuild),
"MODULES_OVERRIDE='{0}'".format(modules),
"buildkernel",
log=log
)
def clean_ufs_image():
sh('${BUILD_ROOT}/build/customize/remove-bits.py ${INSTUFS_DESTDIR}')
# Strip binaries
for root, dirs, files in os.walk(e('${INSTUFS_DESTDIR}/')):
for name in files:
filename = os.path.join(root, name)
if os.path.splitext(name)[1] == '.ko':
continue
if not is_elf(filename):
continue
# We need to remove any flags on protected files and restore
# them after stripping
flags = os.stat(filename).st_flags
os.chflags(filename, 0)
sh('strip ${filename}')
os.chflags(filename, flags)
def main():
ref_date = 0
rel_dir = ''
dirstring = e('${BE_ROOT}/release/${PRODUCT}')
for x in glob.glob("{0}*".format(dirstring)):
if os.path.isdir(e('${x}/${BUILD_ARCH_SHORT}')) and os.lstat(x).st_ctime > ref_date:
ref_date = os.lstat(x).st_ctime
rel_dir = x
if not rel_dir:
error('Release not found')
if e('${BUILD_TYPE}').lower() in ["master", "stable"]:
appending_path = e('${VERSION_NUMBER}/${BUILD_TYPE}')
else:
appending_path = e('${VERSION_NUMBER}')
internal_path = os.path.join(e('${IX_INTERNAL_PATH_PREFIX}'), appending_path)
if os.path.exists(internal_path):
internal_path = os.path.join(internal_path, os.path.basename(rel_dir))
sh('cp -r ${rel_dir} ${internal_path}')
def create_ports_list():
info('Creating ports list')
sh('rm -rf', portoptions)
f = open(portslist, 'w')
for port in installer_ports['ports'] + config['ports']:
name = port['name'] if isinstance(port, dict) else port
name_und = name.replace('/', '_')
options_path = pathjoin(portoptions, name_und)
f.write('{0}\n'.format(name))
sh('mkdir -p', options_path)
if isinstance(port, dict) and 'options' in port:
opt = open(pathjoin(options_path, 'options'), 'w')
for o in port['options']:
opt.write('{0}\n'.format(o))
opt.close()
f.close()
def installkernel(kconf, destdir, log, kodir=None, modules=None, conf="build"):
info('Installing kernel in {0}', log)
info('Log file: {0}', log)
if modules is None:
modules = config['kernel_modules']
if kodir is None:
kodir = "/boot/kernel"
makeconf = objdir("make-${conf}.conf")
modules = ' '.join(modules)
sh(
"env MAKEOBJDIRPREFIX=${OBJDIR}",
"make",
"-C ${TRUEOS_ROOT}",
"installkernel",
"DESTDIR=${destdir}",
"KERNCONF={0}".format(kconf),
"KODIR={0}".format(kodir),
"__MAKE_CONF=${makeconf}",
"MODULES_OVERRIDE='{0}'".format(modules),
log=log
)
def main():
pkg_paths = []
pkg_names = []
def append_packages(name):
for i in glob('${OBJDIR}/ports/packages/*/All/'+'{0}*.txz'.format(name)):
pkg_paths.append(i)
pkg_names.append(os.path.basename(i))
if 'install_latest' in sys.argv:
for i in glob('${OBJDIR}/ports/logs/bulk/*/latest/logs/*.log'):
current_pkg = os.path.basename(i).rsplit('.log')[0]
append_packages(current_pkg)
else:
append_packages('${package}')
pkg_dest_paths = ' '.join([os.path.join('/tmp', i) for i in pkg_names])
pkg_paths = ' '.join(pkg_paths)
pkg_names = ' '.join(pkg_names)
sh('scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ${pkg_paths} ${host}:/tmp/')
sh('ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -t ${host} pkg add -f ${pkg_dest_paths}')
def install_quagga():
dist.install("autoconf", "automake", "libtool", "make", "gcc", "gawk", "pkg-config")
if dist.NAME == "Ubuntu" or dist.NAME == "Debian":
dist.install("libreadline-dev", "libc-ares-dev")
elif dist.NAME == "Fedora":
dist.install("readline-devel", "c-ares-devel")
quagga_src = os.path.join(args.output_dir, "quagga-%s" % QuaggaVersion)
quagga_tar = quagga_src + ".tar.gz"
sh("wget http://download.savannah.gnu.org/releases/quagga/quagga-%s.tar.gz" % QuaggaVersion,
"tar -zxvf '%s'" % quagga_tar,
cwd=args.output_dir)
quagga_install = os.path.join(args.output_dir, "quagga")
sh("./configure '--prefix=%s'" % quagga_install,
"make",
"make install",
cwd=quagga_src)
sh("rm -r '%s' '%s'" % (quagga_src, quagga_tar))
sh("groupadd quagga", may_fail=True)
sh("usermod -a -G quagga root", may_fail=True)
for root, _, files in os.walk(os.path.join(quagga_install, "sbin")):
for f in files:
link = os.path.join("/usr/sbin", os.path.basename(f))
if os.path.exists(link):
os.remove(link)
os.symlink(os.path.join(root, f), link)
break
for root, _, files in os.walk(os.path.join(quagga_install, "bin")):
for f in files:
link = os.path.join("/usr/bin", os.path.basename(f))
if os.path.exists(link):
os.remove(link)
os.symlink(os.path.join(root, f), link)
break
def obtain_jail_name():
global jailname
for i in string.ascii_lowercase:
user = e('${SUDO_USER}')
if user:
i = e('${i}-${user}')
if sh('jls -q -n -j j${i}-p', log="/dev/null", nofail=True) != 0:
jailname = e('j${i}')
setfile(e('${OBJDIR}/jailname'), jailname)
return
error('No jail names available')
def merge_port_trees():
for i in config['port_trees']:
info(e('Merging ports tree ${i}'))
for p in glob('${i}/*/*'):
portpath = '/'.join(p.split('/')[-2:])
sh('rm -rf ${PORTS_OVERLAY}/${portpath}')
sh('mkdir -p ${PORTS_OVERLAY}/${portpath}')
sh('cp -lr ${p}/ ${PORTS_OVERLAY}/${portpath}')
# POSSIBILITY OF SUCH DAMAGE.
#
#####################################################################
import os
import sys
from utils import sh, e, setup_env, objdir, info, import_function
installworldlog = objdir('logs/jail-installworld')
distributionlog = objdir('logs/jail-distribution')
installkernellog = objdir('logs/jail-installkernel')
installworld = import_function('build-os', 'installworld')
installkernel = import_function('build-os', 'installkernel')
if __name__ == '__main__':
if e('${SKIP_INSTALL_JAIL}'):
info(
'Skipping jail installation, as instructed by setting SKIP_INSTALL_JAIL'
)
sys.exit(0)
if os.path.isdir(e('${WORLD_DESTDIR}')):
sh('chflags -fR 0 ${JAIL_DESTDIR}')
sh('rm -rf ${JAIL_DESTDIR}')
sh('mkdir -p ${JAIL_DESTDIR}')
installworld(e('${JAIL_DESTDIR}'),
installworldlog,
distributionlog,
conf="build")
def clean_iso_image():
sh("find ${ISO_DESTDIR}/boot -name '*.symbols' -delete")
def create_overlay():
info('Creating ports overlay...')
sh('rm -rf ${PORTS_OVERLAY}')
sh('mkdir -p ${PORTS_OVERLAY}')
sh('cp -lr ${PORTS_ROOT}/ ${PORTS_OVERLAY}')
def stage_upgrade():
sh('rm -rf ${UPGRADE_STAGEDIR}')
sh('mkdir -p ${UPGRADE_STAGEDIR}')
sh('cp -R ${OBJDIR}/packages/Packages ${UPGRADE_STAGEDIR}/')
# If RESTART is given, save that
if env('RESTART'):
sh('echo ${RESTART} > ${UPGRADE_STAGEDIR}/RESTART')
# And if REBOOT is given, put that in FORCEREBOOT
if env('REBOOT'):
sh('echo ${REBOOT} > ${UPGRADE_STAGEDIR}/FORCEREBOOT')
sh('rm -f ${BE_ROOT}/release/LATEST')
sh('ln -sf ${UPGRADE_STAGEDIR} ${BE_ROOT}/release/LATEST')
def create_iso_dirs():
sh('mkdir -p ${ISO_DESTDIR}/data')
sh('mkdir -p ${ISO_DESTDIR}/dev')
sh('mkdir -p ${ISO_DESTDIR}/.mount')
sh('mkdir -p ${ISO_DESTDIR}/mnt')
sh('mkdir -p ${ISO_DESTDIR}/tmp')
sh('mkdir -p ${ISO_DESTDIR}/boot/grub')
def create_ufs_dirs():
sh('mkdir -p ${INSTUFS_DESTDIR}/usr/local/pre-install')
sh('mkdir -p ${INSTUFS_DESTDIR}/usr/local/firmware')
sh('mkdir -p ${INSTUFS_DESTDIR}/usr/local/install')
sh('mkdir -p ${INSTUFS_DESTDIR}/usr/local/sbin')
sh('mkdir -p ${INSTUFS_DESTDIR}/.mount')
sh('mkdir -p ${INSTUFS_DESTDIR}/cdrom')
sh('mkdir -p ${INSTUFS_DESTDIR}/conf/default/etc')
sh('mkdir -p ${INSTUFS_DESTDIR}/conf/default/tmp')
sh('mkdir -p ${INSTUFS_DESTDIR}/conf/default/var')
sh('mkdir -p ${INSTUFS_DESTDIR}/tank')
sh('rm -rf ${INSTUFS_DESTDIR}/boot')
def merge_port_trees():
for i in config['port_trees']:
info(e('Merging ports tree ${i}'))
uids = "%s/%s" % (i, "UIDs")
gids = "%s/%s" % (i, "GIDs")
for p in glob('${i}/*/*'):
portpath = '/'.join(p.split('/')[-2:])
if portpath.startswith('Mk'):
if os.path.isdir(e('${PORTS_OVERLAY}/${portpath}')):
sh('cp -lf ${p}/* ${PORTS_OVERLAY}/${portpath}/')
else:
sh('cp -l ${p} ${PORTS_OVERLAY}/${portpath}')
else:
sh('rm -rf ${PORTS_OVERLAY}/${portpath}')
sh('mkdir -p ${PORTS_OVERLAY}/${portpath}')
sh('cp -lr ${p}/ ${PORTS_OVERLAY}/${portpath}')
if os.path.exists(uids):
sh('rm -f ${PORTS_OVERLAY}/UIDs')
sh('cp -l ${uids} ${PORTS_OVERLAY}/UIDs')
if os.path.exists(gids):
sh('rm -rf ${PORTS_OVERLAY}/GIDs')
sh('cp -l ${gids} ${PORTS_OVERLAY}/GIDs')
def copy_packages():
sh('mkdir -p ${ISO_DESTDIR}/${PRODUCT}')
sh('cp -R ${OBJDIR}/packages/Packages ${ISO_DESTDIR}/${PRODUCT}')
sh('cp ${OBJDIR}/packages/${PRODUCT}-MANIFEST ${ISO_DESTDIR}/')
def main(destdir):
# Kill docs
sh('rm -rf ${destdir}/usr/local/share/doc')
sh('rm -rf ${destdir}/usr/local/share/gtk-doc')
# Kill gobject introspection xml
sh('rm -rf ${destdir}/usr/local/share/git-1.0')
# Kill examples
sh('rm -rf ${destdir}/usr/local/share/examples')
sh('rm -rf ${destdir}/usr/share/me')
# Kill sources of locale files
sh("find ${destdir}/usr/local -type f -name '*.po' -delete")
# Kill install leftover (#27013)
sh('rm -rf ${destdir}/var/tmp/rc.conf.frenas')
sh('rm -rf ${destdir}/var/tmp/freenas_config.md5')
# magic.mgc is just a speed optimization
sh('rm -f ${destdir}/usr/share/misc/magic.mgc')
# If we are doing SDK build, we can stop here
if e('${SDK}') == "yes":
info('SDK: Skipping remove-bits...')
return 0
# Kill static libraries
sh("find ${destdir}/usr/local \( -name '*.a' -or -name '*.la' \) -delete")
sh("rm -rf ${destdir}/usr/lib/*.a")
# Kill info
sh('rm -rf ${destdir}/usr/local/info')
# Kill .pyo files
sh("find ${destdir}/usr/local \( -name '*.pyo' \) -delete")
# We don't need python test in the image
sh('rm -rf ${destdir}/usr/local/lib/python3.6/test')
# Kill includes
sh("find ${destdir}/usr/local/include \( \! -name 'pyconfig.h' \) -type f -delete"
)
def stage_upgrade():
sh('rm -rf ${UPGRADE_STAGEDIR}')
sh('mkdir -p ${UPGRADE_STAGEDIR}')
sh('cp -R ${OBJDIR}/packages/Packages ${UPGRADE_STAGEDIR}/')
# Move any validation scripts back
for v in "ValidateInstall", "ValidateUpdate":
if os.path.exists(e('${UPGRADE_STAGEDIR}/Packages/${v}')):
sh(e("mv ${UPGRADE_STAGEDIR}/Packages/${v} ${UPGRADE_STAGEDIR}/${v}"))
# If RESTART is given, save that
if env('RESTART'):
sh('echo ${RESTART} > ${UPGRADE_STAGEDIR}/RESTART')
# And if REBOOT is given, put that in FORCEREBOOT
if env('REBOOT'):
sh('echo ${REBOOT} > ${UPGRADE_STAGEDIR}/FORCEREBOOT')
sh('rm -f ${BE_ROOT}/release/LATEST')
sh('ln -sf ${UPGRADE_STAGEDIR} ${BE_ROOT}/release/LATEST')
def mount_packages():
on_abort(umount_packages)
jailname = readfile(e('${OBJDIR}/jailname'))
sh('mkdir -p ${WORLD_DESTDIR}/usr/ports/packages')
sh('mount -t nullfs ${OBJDIR}/ports/data/packages/${jailname}-p ${WORLD_DESTDIR}/usr/ports/packages'
)
from utils import sh, e, objdir, info, import_function
installworldlog = objdir('logs/dest-installworld')
distributionlog = objdir('logs/dest-distribution')
installkernellog = objdir('logs/dest-installkernel')
installkerneldebuglog = objdir('logs/dest-installkerneldebug')
installworld = import_function('build-os', 'installworld')
installkernel = import_function('build-os', 'installkernel')
if __name__ == '__main__':
if e('${SKIP_INSTALL_WORLD}'):
info('Skipping world installation, as instructed by setting SKIP_INSTALL_WORLD')
sys.exit(0)
if os.path.isdir(e('${WORLD_DESTDIR}')):
sh('chflags -fR 0 ${WORLD_DESTDIR}')
sh('rm -rf ${WORLD_DESTDIR}')
sh('mkdir -p ${WORLD_DESTDIR}')
installworld(e('${WORLD_DESTDIR}'), installworldlog, distributionlog, conf="run")
installkernel(e('${KERNCONF}'), e('${WORLD_DESTDIR}'), installkernellog, conf="run")
installkernel(
e('${KERNCONF}-DEBUG'),
e('${WORLD_DESTDIR}'),
installkerneldebuglog,
kodir="/boot/kernel-debug",
conf="run"
)
def main():
for i in dsl['early_customize_tasks']:
logfile = objdir('logs/custom-${i}')
sh('${BUILD_ROOT}/build/customize/${i}.py ${WORLD_DESTDIR}',
log=logfile)
def clean_ufs_image():
sh('${BUILD_ROOT}/build/customize/remove-bits.py ${INSTUFS_DESTDIR}')
def copy_data():
sh('mkdir -p ${INSTUFS_DESTDIR}/data')
sh('cp -R ${WORLD_DESTDIR}/data ${INSTUFS_DESTDIR}/')
def main():
sh('mkdir -p ${WORLD_DESTDIR}/conf/base/etc')
sh('mkdir -p ${WORLD_DESTDIR}/conf/base/var')
sh('mkdir -p ${WORLD_DESTDIR}/conf/base/mnt')
sh('touch ${WORLD_DESTDIR}/etc/diskless')
sh('cp -a ${WORLD_DESTDIR}/etc/ ${WORLD_DESTDIR}/conf/base/etc')
sh('cp -a ${WORLD_DESTDIR}/var/ ${WORLD_DESTDIR}/conf/base/var')
setfile('${WORLD_DESTDIR}/conf/base/var/md_size', '')
setfile('${WORLD_DESTDIR}/conf/base/etc/md_size', '65535')
setfile('${WORLD_DESTDIR}/conf/base/mnt/md_size', '8192')
def umount_packages():
sh('umount ${INSTUFS_DESTDIR}/usr/ports/packages')
on_abort(None)
def tag_repo(repo, tag):
sh("git --git-dir=${repo}/.git tag ${tag}")
def install_pkgtools():
info('Installing freenas-pkgtools')
sh(
"env MAKEOBJDIRPREFIX=${OBJDIR}",
"make -C ${SRC_ROOT}/freenas-pkgtools obj all install DESTDIR=${INSTUFS_DESTDIR} PREFIX=/usr/local",
)
def checkout_repo(cwd, repo):
"""Check out the given repository.
Arguments:
cwd -- start in this directory.
repo -- gives 'name', 'path', 'branch', and 'url'
(and optionally 'commit')
We check out the given branch, unless ${CHECKOUT_TAG} is
set (then we check out that value), or unless a 'commit'
key is set (then we check out repo['commit']).
If ${CHECKOUT_SHALLOW} is set, new clones are made with
depth 1.
If ${GIT_REF_PATH} is set, we can check for reference clones
that may be available in that path (colon separated path
as for normal Unix conventions).
"""
buildenv_root = e('${BE_ROOT}')
repo_name = repo['name']
repo_path = repo['path']
repo_url = repo['url']
branch = repo['branch']
# Search for a reference clone before changing directories
# in case it's a relative path.
os.chdir(cwd)
refclone = find_ref_clone(repo_name)
if refclone:
refclone = os.path.abspath(refclone)
os.chdir(buildenv_root)
if is_git_repo(repo_path):
os.chdir(repo_path)
current_branch = sh_str('git rev-parse --abbrev-ref HEAD')
if current_branch != branch:
# (re)setting origin is a bit rude if someone had
# carefully set their own variant, but oh well.
sh('git remote set-url origin', repo_url)
sh('git fetch origin')
sh('git checkout', branch)
# git pull --rebase exhibits bad behavior in git 2.8.x and
# early 2.9.x, leaving dead lock files. This is an attempted
# work-around - it should behave the same, perhaps minus
# internal git bugs.
sh('git fetch && git rebase')
else:
if e('${CHECKOUT_SHALLOW}'):
sh('git clone -b', branch, '--depth 1', repo_url, repo_path)
else:
# Should we have an option to add --dissociate?
if refclone:
sh('git clone --reference', refclone, '-b', branch, repo_url,
repo_path)
else:
sh('git clone -b', branch, repo_url, repo_path)
os.chdir(repo_path)
if e('${CHECKOUT_TAG}'):
sh('git checkout ${CHECKOUT_TAG}')
elif 'commit' in repo:
sh('git checkout', repo['commit'])
manifest[repo_url] = get_git_rev()
def setup_diskless():
sh('touch ${INSTUFS_DESTDIR}/etc/diskless')
sh('cp -a ${INSTUFS_DESTDIR}/etc/ ${INSTUFS_DESTDIR}/conf/default/etc')
sh('cp -a ${INSTUFS_DESTDIR}/var/ ${INSTUFS_DESTDIR}/conf/default/var')
def generate_manifest():
sh('rm -f ${BE_ROOT}/repo-manifest')
for k, v in manifest.items():
appendfile('${BE_ROOT}/repo-manifest', e('${k} ${v}'))
def main():
prod = e("${PRODUCTION}")
if prod and prod.lower() == "yes":
KEY_PASSWORD = getpass.getpass("Enter Password: "******"Hey Release Engineer this is the password you entered: {0}".format(KEY_PASSWORD))
else:
KEY_PASSWORD = ""
changelog = e('${CHANGELOG}')
ssh = e('${UPDATE_USER}@${UPDATE_HOST}')
sshopts='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
temp_dest = sh_str("ssh ${ssh} ${sshopts} mktemp -d /tmp/update-${PRODUCT}-XXXXXXXXX")
temp_changelog = sh_str("ssh ${ssh} ${sshopts} mktemp /tmp/changelog-XXXXXXXXX")
if not temp_dest or not temp_changelog:
error('Failed to create temporary directories on {0}', ssh)
sh('scp ${sshopts} -r ${BE_ROOT}/release/LATEST/. ${ssh}:${temp_dest}')
if changelog:
cl_file = None
if changelog == '-':
print('Enter changelog, ^D to end:')
cl_file = tempfile.NamedTemporaryFile(delete=False)
cl_file.write(bytes(sys.stdin.read(),'UTF-8'))
cl_file.close()
changelog = cl_file.name
sh('scp ${sshopts} ${changelog} ${ssh}:${temp_changelog}')
if cl_file is not None:
os.remove(cl_file.name)
sh(
"echo ${KEY_PASSWORD} |",
"ssh ${sshopts} ${ssh}",
"/usr/local/bin/freenas-release",
"-P ${PRODUCT}",
"-D ${UPDATE_DB}",
"--archive ${UPDATE_DEST}",
"-K ${FREENAS_KEYFILE}",
"-C ${temp_changelog}" if changelog else "",
"add ${temp_dest}"
)
sh("ssh ${sshopts} ${ssh} rm -rf ${temp_dest}")
sh("ssh ${sshopts} ${ssh} rm -rf ${temp_changelog}")
# This last line syncs up with the cdn
# It is only done in the case of public facing update
if e("${INTERNAL_UPDATE}").lower() == "no":
sh("ssh ${sshopts} ${ssh} /usr/local/sbin/rsync-mirror.sh")
def checkout_repo(repo):
os.chdir(e('${BE_ROOT}'))
if os.path.isdir(os.path.join(repo['path'], '.git')):
os.chdir(repo['path'])
branch = sh_str('git rev-parse --abbrev-ref HEAD')
if branch != repo['branch']:
sh('git remote set-url origin', repo['url'])
sh('git fetch origin')
sh('git checkout', repo['branch'])
sh('git pull --rebase')
else:
if e('${CHECKOUT_SHALLOW}'):
sh('git clone', '-b', repo['branch'], '--depth', '1', repo['url'], repo['path'])
else:
sh('git clone', '-b', repo['branch'], repo['url'], repo['path'])
os.chdir(repo['path'])
if e('${CHECKOUT_TAG}'):
sh('git checkout ${CHECKOUT_TAG}')
elif 'commit' in repo:
sh('git checkout', repo['commit'])
manifest[repo['url']] = sh_str("git rev-parse --short HEAD")
def main():
user = sh_str('id -un')
if user == 'root':
user = '******'
ref_date = 0
rel_dir = ''
dirstring = e('${BE_ROOT}/release/${PRODUCT}')
for x in glob.glob("{0}*".format(dirstring)):
if os.path.isdir(e('${x}/${BUILD_ARCH_SHORT}')
) and os.lstat(x).st_ctime > ref_date:
ref_date = os.lstat(x).st_ctime
rel_dir = x
if not rel_dir:
error('Release not found')
download_suffix = ""
if e('${BUILD_TYPE}').lower() in [
"master", "stable", "freebsd-stable", "freebsd-head"
]:
buildtimestamp = os.path.basename(rel_dir).split("-")[-1]
download_suffix = e('${BUILD_TYPE}/${buildtimestamp}')
else:
download_suffix = e('${BUILD_TYPE}')
downloadtargetdir = e('${DOWNLOAD_PREFIX}/${download_suffix}')
sh('ssh ${user}@${DOWNLOAD_HOST} rm -rf ${downloadtargetdir}')
sh('ssh ${user}@${DOWNLOAD_HOST} mkdir -p ${downloadtargetdir}')
sh('scp -pr ${rel_dir}/* ${user}@${DOWNLOAD_HOST}:${downloadtargetdir}/')
# For all non-nightly builds create latest symlinks
if e('${BUILD_TYPE}').lower() in [
"master", "freebsd-stable", "freebsd-head"
]:
info('Creating MILESTONE level downloads symlink')
sh('ssh ${user}@${DOWNLOAD_HOST} ln -shf ${buildtimestamp} ${DOWNLOAD_PREFIX}/${BUILD_TYPE}/latest'
)
else:
info('Creating top level downloads symlink')
sh('ssh ${user}@${DOWNLOAD_HOST} ln -shf ${VERSION_NUMBER}/${download_suffix} ${DOWNLOAD_BASEDIR}/latest'
)
info('Synchronizing download server to CDN')
sh('ssh ${user}@${DOWNLOAD_HOST} /usr/local/sbin/rsync-mirror.sh')
def make_ufs_image():
sh('mkdir -p ${ISO_DESTDIR}/data')
sh('makefs -b 10% ${imgfile} ${INSTUFS_DESTDIR}')
sh('mkuzip -o ${ISO_DESTDIR}/data/base.ufs.uzip ${imgfile}')
def umount_packages():
sh('umount ${WORLD_DESTDIR}/usr/ports/packages')
on_abort(None)
def prepare_env():
for cmd in jailconf.get('copy', []):
dest = os.path.join(e('${JAIL_DESTDIR}'), cmd['dest'][1:])
sh('rm -rf ${dest}')
sh('cp -a', cmd['source'], dest)
for cmd in jailconf.get('link', []):
flags = '-o {0}'.format(cmd['flags']) if 'flags' in cmd else ''
dest = os.path.join(e('${JAIL_DESTDIR}'), cmd['dest'][1:])
sh('mkdir -p', os.path.dirname(dest))
sh('mount -t nullfs', flags, cmd['source'], dest)
osversion = sh_str(
"awk '/\#define __FreeBSD_version/ { print $3 }' ${JAIL_DESTDIR}/usr/include/sys/param.h"
)
login_env = e(
',UNAME_r=${FREEBSD_RELEASE_VERSION% *},UNAME_v=FreeBSD ${FREEBSD_RELEASE_VERSION},OSVERSION=${osversion}'
)
sh('sed -i "" -e "s/,UNAME_r.*:/:/ ; s/:\(setenv.*\):/:\\1${login_env}:/" ${JAIL_DESTDIR}/etc/login.conf'
)
sh('cap_mkdb ${JAIL_DESTDIR}/etc/login.conf')
if e('${USE_ZFS}'):
sh('zfs snapshot ${ZPOOL}${ZROOTFS}/jail@clean')
def create_pkgng_configuration():
sh('mkdir -p ${WORLD_DESTDIR}/usr/local/etc/pkg/repos')
for i in glob.glob(e('${BUILD_CONFIG}/templates/pkg-repos/*')):
fname = os.path.basename(i)
sh(e('cp ${i} ${WORLD_DESTDIR}/usr/local/etc/pkg/repos/${fname}'))
def main(destdir):
# Kill .pyo files
sh("find ${destdir}/usr/local \( -name '*.pyo' \) -delete")
# Kill includes
sh("find ${destdir}/usr/local/include \( \! -name 'pyconfig.h' \) -type f -delete"
)
# Kill docs
sh('rm -rf ${destdir}/usr/local/share/doc')
sh('rm -rf ${destdir}/usr/local/share/gtk-doc')
# Kill gobject introspection xml
sh('rm -rf ${destdir}/usr/local/share/git-1.0')
# Kill info
sh('rm -rf ${destdir}/usr/local/info')
# Kill man pages
sh('rm -rf ${destdir}/usr/local/man')
# Kill examples
sh('rm -rf ${destdir}/usr/local/share/examples')
# Kill groff_fonts junk
sh('rm -rf ${destdir}/usr/share/groff_font')
sh('rm -rf ${destdir}/usr/share/tmac')
sh('rm -rf ${destdir}/usr/share/me')
# Kill static libraries
sh("find ${destdir}/usr/local \( -name '*.a' -or -name '*.la' \) -delete")
# magic.mgc is just a speed optimization
sh('rm -f ${destdir}/usr/share/misc/magic.mgc')
