def build(self, dir_=None):
"""Build the debian package.
When this method is called it is assumed that all file
prerequisites for the package have already been built
or met. This is simply the packaging stage.
'dir_' : This is the output directory in which the package
should be left. If unspecified the package's local directory
will contain the package file."""
if 'external' in self.pkg:
# Package file built externally
epkg = self.pkg['external']
if os.path.exists(epkg):
return epkg
else:
raise OnlPackageError("The external package file '%s' does not exist." % epkg);
# Make sure all required files exist
if 'files' in self.pkg:
self._validate_files('files', True)
if 'optional-files' in self.pkg:
self._validate_files('optional-files', False)
# If dir_ is not specified, leave package in local package directory.
if dir_ is None:
dir_ = self.dir
workdir = tempfile.mkdtemp()
root = os.path.join(workdir, "root");
os.mkdir(root);
# The package file will be built into the workdir
self.pkg['__workdir'] = workdir
for (src,dst) in self.pkg.get('files', {}):
OnlPackage.copyf(src, dst, root)
for (src,dst) in self.pkg.get('optional-files', {}):
if os.path.exists(src):
OnlPackage.copyf(src, dst, root)
for (link,src) in self.pkg.get('links', {}).iteritems():
logger.info("Linking %s -> %s..." % (link, src))
link = os.path.join(root, link)
os.symlink(src, link)
#
# FPM doesn't seem to have a doc option so we copy documentation
# files directly into place.
#
docpath = os.path.join(root, "usr/share/doc/%(name)s" % self.pkg)
if not os.path.exists(docpath):
os.makedirs(docpath)
for src in self.pkg.get('docs', []):
if not os.path.exists(src):
raise OnlPackageError("Documentation source file '%s' does not exist." % src)
shutil.copy(src, docpath)
changelog = os.path.join(workdir, 'changelog')
copyright_ = os.path.join(workdir, 'copyright')
#
# Export changelog and copyright files from the PKG dict
# to the workdir for processing.
#
# The copyright and changelog data can be embedded directly
# int the PKG file or kept as separate files.
#
def copy_str_or_file(src, dst):
if os.path.exists(src):
shutil.copyfile(src, dst)
else:
with open(dst, "w") as f:
f.write(src)
f.write("\n")
copy_str_or_file(self.pkg['copyright'], copyright_)
copy_str_or_file(self.pkg['changelog'], changelog)
############################################################
#
# Invoke fpm with all necessary options.
#
############################################################
self.pkg['__root'] = root
command = """fpm -p %(__workdir)s -f -C %(__root)s -s dir -t deb -n %(name)s -v %(version)s -a %(arch)s -m %(maintainer)s --description "%(description)s" --url "%(url)s" --license "%(license)s" --vendor "%(vendor)s" """ % self.pkg
for dep in self.pkg.get('depends', []):
command = command + "-d %s " % dep
for provides in onlu.sflatten(self.pkg.get('provides', [])):
command = command + "--provides %s " % provides
for conflicts in onlu.sflatten(self.pkg.get('conflicts', [])):
command = command + "--conflicts %s " % conflicts
for replaces in onlu.sflatten(self.pkg.get('replaces', [])):
command = command + "--replaces %s " % replaces
if 'virtual' in self.pkg:
command = command + "--provides %(v)s --conflicts %(v)s --replaces %(v)s " % dict(v=self.pkg['virtual'])
if 'priority' in self.pkg:
command = command + "--deb-priority %s " % self.pkg['priority']
if 'init' in self.pkg:
if not os.path.exists(self.pkg['init']):
raise OnlPackageError("Init script '%s' does not exist." % self.pkg['init'])
command = command + "--deb-init %s " % self.pkg['init']
if self.pkg.get('init-after-install', True):
command = command + "--after-install %s " % OnlPackageAfterInstallScript(self.pkg['init'], dir=workdir).name
if self.pkg.get('init-before-remove', True):
command = command + "--before-remove %s " % OnlPackageBeforeRemoveScript(self.pkg['init'], dir=workdir).name
if self.pkg.get('init-after-remove', True):
command = command + "--after-remove %s " % OnlPackageAfterRemoveScript(self.pkg['init'], dir=workdir).name
if self.pkg.get('asr', False):
with onlu.Profiler() as profiler:
# Generate the ASR documentation for this package.
sys.path.append("%s/sm/infra/tools" % os.getenv('ONL'))
import asr
asro = asr.AimSyslogReference()
asro.extract(workdir)
asro.format(os.path.join(docpath, asr.AimSyslogReference.ASR_NAME), 'json')
profiler.log("ASR generation for %(name)s" % self.pkg)
############################################################
if logger.level < logging.INFO:
command = command + "--verbose "
onlu.execute(command)
# Grab the package from the workdir. There can be only one.
files = glob.glob(os.path.join(workdir, '*.deb'))
if len(files) == 0:
raise OnlPackageError("No debian package.")
elif len(files) > 1:
raise OnlPackageError("Too many packages.")
else:
# Move to the target directory
shutil.copy(files[0], dir_)
# Remove entire work directory.
shutil.rmtree(workdir)
# Return the path to the final built package
return os.path.join(dir_, os.path.basename(files[0]))
def configure(self, dir_):
if not os.getenv('NO_DPKG_CONFIGURE'):
with OnlRfsContext(dir_, resolvconf=False):
self.dpkg_configure(dir_)
with OnlRfsContext(dir_):
os_release = os.path.join(dir_, 'etc', 'os-release')
if os.path.exists(os_release):
# Convert /etc/os-release to /etc/os-release.json
import shlex
contents = open(os_release).read()
d = dict(token.split('=') for token in shlex.split(contents))
ua = OnlRfsSystemAdmin(dir_)
ua.chmod('a+rwx', os.path.dirname(os_release))
with open(
os.path.join(os.path.dirname(os_release),
'os-release.json'), "w") as f:
f.write(json.dumps(d))
ua.chmod('0755', os.path.dirname(os_release))
Configure = self.config.get('Configure', None)
if Configure:
for cmd in Configure.get('run', []):
onlu.execute("sudo chroot %s %s" % (dir_, cmd),
ex=OnlRfsError("run command '%s' failed" %
cmd))
for overlay in Configure.get('overlays', []):
logger.info("Overlay %s..." % overlay)
onlu.execute(
'tar -C %s -c --exclude "*~" . | sudo tar -C %s -x -v --no-same-owner'
% (overlay, dir_),
ex=OnlRfsError("Overlay '%s' failed." % overlay))
for update in Configure.get('update-rc.d', []):
onlu.execute("sudo chroot %s /usr/sbin/update-rc.d %s" %
(dir_, update),
ex=OnlRfsError("update-rc.d %s failed." %
(update)))
for script in Configure.get('scripts', []):
logger.info("Configuration script %s..." % script)
onlu.execute("sudo %s %s" % (script, dir_),
ex=OnlRfsError("script '%s' failed." %
script))
for command in Configure.get('commands', []):
if '__rfs__' in command:
command = command % dict(__rfs__=dir_)
logger.info("Configuration command '%s'..." % command)
onlu.execute(command,
ex=OnlRfsError("Command '%s' failed." %
command))
ua = OnlRfsSystemAdmin(dir_)
for (group, values) in Configure.get('groups', {}).iteritems():
ua.groupadd(group=group, **values if values else {})
for (user, values) in Configure.get('users', {}).iteritems():
if user == 'root':
if 'password' in values:
ua.user_password_set(user, values['password'])
else:
ua.useradd(username=user, **values)
options = Configure.get('options', {})
if options.get('clean', False):
logger.info("Cleaning Filesystem...")
onlu.execute('sudo chroot %s /usr/bin/apt-get clean' %
dir_)
onlu.execute('sudo chroot %s /usr/sbin/localepurge' % dir_)
onlu.execute(
'sudo chroot %s find /usr/share/doc -type f -not -name asr.json -delete'
% dir_)
onlu.execute(
'sudo chroot %s find /usr/share/man -type f -delete' %
dir_)
if 'PermitRootLogin' in options:
config = os.path.join(dir_, 'etc/ssh/sshd_config')
ua.chmod('a+rw', config)
lines = open(config).readlines()
with open(config, "w") as f:
for line in lines:
if line.startswith('PermitRootLogin'):
v = options['PermitRootLogin']
logger.info("Setting PermitRootLogin to %s" %
v)
f.write('PermitRootLogin %s\n' % v)
else:
f.write(line)
ua.chmod('644', config)
if not options.get('securetty', True):
f = os.path.join(dir_, 'etc/securetty')
if os.path.exists(f):
logger.info("Removing %s" % f)
onlu.execute('sudo rm %s' % f,
ex=OnlRfsError(
'Could not remove file %s' % f))
if not options.get('ttys', False):
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
logger.info("Clearing %s ttys..." % f)
for line in fileinput.input(f, inplace=True):
if re.match("^[123456]:.*", line):
line = "#" + line
print line,
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('console', True):
logger.info('Configuring Console Access in %s' % f)
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
with open(f, 'a') as h:
h.write("T0:23:respawn:/sbin/pgetty\n")
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('asr', None):
asropts = options.get('asr')
logger.info("Gathering ASR documentation...")
sys.path.append("%s/sm/infra/tools" % os.getenv('ONL'))
import asr
asro = asr.AimSyslogReference()
asro.merge(dir_)
asro.format(os.path.join(dir_, asropts['file']),
fmt=asropts['format'])
for (mf, fields) in Configure.get('manifests', {}).iteritems():
logger.info("Configuring manifest %s..." % mf)
if mf.startswith('/'):
mf = mf[1:]
mname = os.path.join(dir_, mf)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(mname))
onlu.execute("sudo touch %s" % mname)
onlu.execute("sudo chmod a+w %s" % mname)
md = {}
md['version'] = json.load(open(fields['version']))
md['arch'] = self.arch
if os.path.exists(fields['platforms']):
md['platforms'] = yaml.load(open(fields['platforms']))
else:
md['platforms'] = fields['platforms'].split(',')
for (k, v) in fields.get('keys', {}).iteritems():
if k in md:
md[k].update(v)
else:
md[k] = v
with open(mname, "w") as f:
json.dump(md, f, indent=2)
onlu.execute("sudo chmod a-w %s" % mname)
for (fname, v) in Configure.get('files',
{}).get('add', {}).iteritems():
if fname.startswith('/'):
fname = fname[1:]
dst = os.path.join(dir_, fname)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(dst))
onlu.execute("sudo touch %s" % dst)
onlu.execute("sudo chmod a+w %s" % dst)
if os.path.exists(v):
shutil.copy(v, dst)
else:
with open(dst, "w") as f:
f.write("%s\n" % v)
for fname in Configure.get('files', {}).get('remove', []):
if fname.startswith('/'):
fname = fname[1:]
f = os.path.join(dir_, fname)
if os.path.exists(f):
onlu.execute("sudo rm -rf %s" % f)
if Configure.get('issue'):
issue = Configure.get('issue')
fn = os.path.join(dir_, "etc/issue")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
fn = os.path.join(dir_, "etc/issue.net")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
def configure(self, dir_):
if not os.getenv('NO_DPKG_CONFIGURE'):
with OnlRfsContext(dir_, resolvconf=False):
self.dpkg_configure(dir_)
with OnlRfsContext(dir_):
os_release = os.path.join(dir_, 'etc', 'os-release')
os_release_dict = {}
if os.path.exists(os_release):
# Convert /etc/os-release to /etc/os-release.json
import shlex
contents = open(os_release).read()
os_release_dict = dict(token.split('=') for token in shlex.split(contents))
ua = OnlRfsSystemAdmin(dir_)
ua.chmod('a+rwx', os.path.dirname(os_release))
with open(os.path.join(os.path.dirname(os_release), 'os-release.json'), "w") as f:
f.write(json.dumps(os_release_dict))
ua.chmod('0755', os.path.dirname(os_release))
Configure = self.config.get('Configure', None)
if Configure:
for cmd in Configure.get('run', []):
onlu.execute("sudo chroot %s %s" % (dir_, cmd),
ex=OnlRfsError("run command '%s' failed" % cmd))
for overlay in Configure.get('overlays', []):
logger.info("Overlay %s..." % overlay)
onlu.execute('tar -C %s -c --exclude "*~" . | sudo tar -C %s -x -v --no-same-owner' % (overlay, dir_),
ex=OnlRfsError("Overlay '%s' failed." % overlay))
for update in Configure.get('update-rc.d', []):
onlu.execute("sudo chroot %s /usr/sbin/update-rc.d %s" % (dir_, update),
ex=OnlRfsError("update-rc.d %s failed." % (update)))
for module in Configure.get('modules', []):
monfig = os.path.join(dir_, 'etc/modules')
ua.chmod('a+rw', monfig)
# This is a bad way to add the modules but works for now
onlu.execute("sudo chroot %s echo %s >> %s" % (dir_, module, monfig))
for sysctl in Configure.get('sysctl', []):
onlu.execute("sudo chroot %s /bin/systemctl %s" % (dir_, sysctl))
for script in Configure.get('scripts', []):
logger.info("Configuration script %s..." % script)
onlu.execute("sudo %s %s" % (script, dir_),
ex=OnlRfsError("script '%s' failed." % script))
for command in Configure.get('commands', []):
if '__rfs__' in command:
command = command % dict(__rfs__=dir_)
logger.info("Configuration command '%s'..." % command)
onlu.execute(command,
ex=OnlRfsError("Command '%s' failed." % command))
ua = OnlRfsSystemAdmin(dir_)
for (group, values) in Configure.get('groups', {}).iteritems():
ua.groupadd(group=group, **values if values else {})
for (user, values) in Configure.get('users', {}).iteritems():
if user == 'root':
if 'password' in values:
ua.user_password_set(user, values['password'])
else:
ua.useradd(username=user, **values)
options = Configure.get('options', {})
if options.get('clean', False):
logger.info("Cleaning Filesystem...")
onlu.execute('sudo chroot %s /usr/bin/apt-get clean' % dir_)
onlu.execute('sudo chroot %s /usr/sbin/localepurge' % dir_ )
onlu.execute('sudo chroot %s find /usr/share/doc -type f -not -name asr.json -delete' % dir_)
onlu.execute('sudo chroot %s find /usr/share/man -type f -delete' % dir_)
if 'PermitRootLogin' in options:
config = os.path.join(dir_, 'etc/ssh/sshd_config')
ua.chmod('a+rw', config)
lines = open(config).readlines()
with open(config, "w") as f:
for line in lines:
# Added the hash sign to update this function
if line.startswith('#PermitRootLogin'):
v = options['PermitRootLogin']
logger.info("Setting PermitRootLogin to %s" % v)
f.write('PermitRootLogin %s\n' % v)
else:
f.write(line)
ua.chmod('644', config)
if not options.get('securetty', True):
f = os.path.join(dir_, 'etc/securetty')
if os.path.exists(f):
logger.info("Removing %s" % f)
onlu.execute('sudo rm %s' % f,
ex=OnlRfsError('Could not remove file %s' % f))
if os.path.exists(os.path.join(dir_, 'etc/inittab')):
if not options.get('ttys', False):
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
logger.info("Clearing %s ttys..." % f)
for line in fileinput.input(f, inplace=True):
if re.match("^[123456]:.*", line):
line = "#" + line
print line,
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('console', True):
logger.info('Configuring Console Access in %s' % f)
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
with open(f, 'a') as h:
h.write("T0:23:respawn:/sbin/pgetty\n")
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('asr', None):
asropts = options.get('asr')
logger.info("Gathering ASR documentation...")
sys.path.append("%s/sm/infra/tools" % os.getenv('ONL'))
import asr
asro = asr.AimSyslogReference()
asro.merge(dir_)
asrf = os.path.join(dir_, asropts['file'])
OnlRfsSystemAdmin.chmod('777', os.path.dirname(asrf))
asro.format(os.path.join(dir_, asropts['file']), fmt=asropts['format'])
for (mf, fields) in Configure.get('manifests', {}).iteritems():
logger.info("Configuring manifest %s..." % mf)
if mf.startswith('/'):
mf = mf[1:]
mname = os.path.join(dir_, mf)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(mname))
onlu.execute("sudo touch %s" % mname)
onlu.execute("sudo chmod a+w %s" % mname)
md = {}
md['version'] = json.load(open(fields['version']))
md['arch'] = self.arch
md['os-release'] = os_release_dict
if os.path.exists(fields['platforms']):
md['platforms'] = yaml.load(open(fields['platforms']))
else:
md['platforms'] = fields['platforms'].split(',')
for (k, v) in fields.get('keys', {}).iteritems():
if k in md:
md[k].update(v)
else:
md[k] = v
with open(mname, "w") as f:
json.dump(md, f, indent=2)
onlu.execute("sudo chmod a-w %s" % mname)
for (fname, v) in Configure.get('files', {}).get('add', {}).iteritems():
if fname.startswith('/'):
fname = fname[1:]
dst = os.path.join(dir_, fname)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(dst))
onlu.execute("sudo touch %s" % dst)
onlu.execute("sudo chmod a+w %s" % dst)
if os.path.exists(v):
shutil.copy(v, dst)
else:
with open(dst, "w") as f:
f.write("%s\n" % v)
for fname in Configure.get('files', {}).get('remove', []):
if fname.startswith('/'):
fname = fname[1:]
f = os.path.join(dir_, fname)
if os.path.exists(f):
onlu.execute("sudo rm -rf %s" % f)
if Configure.get('issue'):
issue = Configure.get('issue')
fn = os.path.join(dir_, "etc/issue")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
fn = os.path.join(dir_, "etc/issue.net")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
if Configure.get('easy_install'):
# install Python2 eggs (with this version of Python)
# tee hee, we cannot use distutils.sysconfig.get_python_lib(),
# because ONL/Dent doesn't use the path layout as the host
pyver = "%d.%d" % (sys.version_info[0], sys.version_info[1],)
pyver = subprocess.check_output(cmd,
universal_newlines=True).strip()
pydir = "%s/usr/lib/python%s/dist-packages" % (dir_, pyver,)
bindir = "%s/usr/bin" % (dir_,)
cmd = "mkdir -p %s" % pydir
onlu.execute(cmd, sudo=True, ex=True)
for egg in Configure.get('easy_install3', []):
logger.info("Installing Egg %s", egg)
cmd = ('easy_install3',
'--always-unzip', '--no-deps',
'--install-dir', pydir,
'--script-dir', bindir,
egg,)
cmd = ("env PYTHONPATH=%s %s"
% (pydir, " ".join(cmd),))
onlu.execute(cmd, sudo=True, ex=True)
if Configure.get('easy_install3'):
# install Python3 eggs (foreign version of Python)
# tee hee, we cannot use distutils.sysconfig.get_python_lib(),
# because ONL/Dent doesn't use the path layout as the host
cmd = "import distutils.sysconfig; print(distutils.sysconfig.get_python_version())"
cmd = ('python3', '-c', cmd,)
pyver = subprocess.check_output(cmd,
universal_newlines=True).strip()
pydir = "%s/usr/lib/python%s/dist-packages" % (dir_, pyver,)
bindir = "%s/usr/bin" % (dir_,)
cmd = "mkdir -p %s" % pydir
onlu.execute(cmd, sudo=True, ex=True)
for egg in Configure.get('easy_install3', []):
logger.info("Installing Egg %s", egg)
cmd = ('easy_install3',
'--always-unzip', '--no-deps',
'--install-dir', pydir,
'--script-dir', bindir,
egg,)
cmd = ("env PYTHONPATH=%s %s"
% (pydir, " ".join(cmd),))
onlu.execute(cmd, sudo=True, ex=True)
