def useradd(self, username, uid, password, shell, deleteFirst=True):
args = [
'useradd', '--non-unique', '--shell', shell, '--home-dir', '/root',
'--uid', '0', '--gid', '0', '--group', 'root'
]
if deleteFirst:
self.userdel(username)
if password:
epassword = crypt.crypt(password, '$1$%s$' % self.gen_salt())
args = args + ['-p', epassword]
args.append(username)
onlu.execute(args,
ex=OnlRfsError("Adding user '%s' failed." % username))
if password is None:
onlu.execute(
"passwd -d %s" % username,
chroot=self.chroot,
ex=OnlRfsError("Error deleting password for user '%s'" %
username))
logger.info("user %s password %s", username, password)
def install(self, dir_, packages):
ONLPM = "%s/tools/onlpm.py" % os.getenv('ONL')
with OnlRfsContext(dir_):
for pspec in packages:
for pkg in pspec.split(','):
cmd = (ONLPM, '--lookup', pkg,)
try:
buf = subprocess.check_output(cmd)
except subprocess.CalledProcessError as ex:
logger.error("cannot find %s", pkg)
raise ValueError("update failed")
if not buf.strip():
raise ValueError("cannot find %s" % pkg)
src = buf.splitlines(False)[0]
d, b = os.path.split(src)
dst = os.path.join(dir_, "tmp", b)
shutil.copy2(src, dst)
src2 = os.path.join("/tmp", b)
logger.info("installing %s into %s", pkg, dir_)
cmd = ('/usr/bin/rfs-dpkg', '-i', src2,)
onlu.execute(cmd,
chroot=dir_,
ex=OnlRfsError("install of %s failed" % pkg))
name, _, _ = pkg.partition(':')
logger.info("updating dependencies for %s", pkg)
cmd = ('/usr/bin/rfs-apt-get', '-f', 'install', name,)
onlu.execute(cmd,
chroot=dir_,
ex=OnlRfsError("install of %s failed" % pkg))
def install(self, dir_, packages):
ONLPM = "%s/tools/onlpm.py" % os.getenv('ONL')
with OnlRfsContext(dir_):
for pspec in packages:
for pkg in pspec.split(','):
cmd = (ONLPM, '--lookup', pkg,)
try:
buf = subprocess.check_output(cmd)
except subprocess.CalledProcessError as ex:
logger.error("cannot find %s", pkg)
raise ValueError("update failed")
if not buf.strip():
raise ValueError("cannot find %s" % pkg)
src = buf.splitlines(False)[0]
d, b = os.path.split(src)
dst = os.path.join(dir_, "tmp", b)
shutil.copy2(src, dst)
src2 = os.path.join("/tmp", b)
logger.info("installing %s into %s", pkg, dir_)
cmd = ('/usr/bin/rfs-dpkg', '-i', src2,)
onlu.execute(cmd,
chroot=dir_,
ex=OnlRfsError("install of %s failed" % pkg))
name, _, _ = pkg.partition(':')
logger.info("updating dependencies for %s", pkg)
cmd = ('/usr/bin/rfs-apt-get', '-f', 'install', name,)
onlu.execute(cmd,
chroot=dir_,
ex=OnlRfsError("install of %s failed" % pkg))
def gmake_locked(self, target, operation):
buildpaths = []
if self._pkgs.get("build", True) and not os.environ.get("NOBUILD", False):
buildpaths = [
os.path.join(self._pkgs["__directory"], "builds"),
os.path.join(self._pkgs["__directory"], "BUILDS"),
]
for bp in buildpaths:
if os.path.exists(bp):
MAKE = os.environ.get("MAKE", "make")
V = " V=1 " if logger.level < logging.INFO else ""
cmd = (
MAKE
+ V
+ " -C "
+ bp
+ " "
+ os.environ.get("ONLPM_MAKE_OPTIONS", "")
+ " "
+ os.environ.get("ONL_MAKE_PARALLEL", "")
+ " "
+ target
)
onlu.execute(cmd, ex=OnlPackageError("%s failed." % operation))
def user_password_set(self, username, password):
logger.info("user %s password now %s", username, password)
epassword = crypt.crypt(password, '$1$%s$' % self.gen_salt())
onlu.execute(['usermod', '-p', epassword, username],
chroot=self.chroot,
ex=OnlRfsError("Error setting password for user '%s'" %
username))
def groupadd(self,
group,
gid=None,
unique=True,
system=False,
force=False,
password=None):
args = ['groupadd']
if force:
args.append("--force")
if system:
args.append("--system")
if not unique:
args.append("--non-unique")
if password:
args = args + ['--password', password]
if gid:
args = args + ['--gid', str(gid)]
args.append(group)
onlu.execute(args,
chroot=self.chroot,
ex=OnlRfsError("Adding group '%s' failed." % group))
logger.info("added group %s", group)
def useradd(self,
username,
uid=None,
gid=None,
password=None,
shell='/bin/bash',
home=None,
groups=None,
sudo=False,
deleteFirst=True):
args = ['useradd', '--create-home']
if uid is not None:
args = args + ['--non-unique', '--uid', str(uid)]
if gid is not None:
args = args + ['--gid', str(gid)]
if password:
epassword = crypt.crypt(password, '$1$%s$' % self.gen_salt())
args = args + ['-p', epassword]
if shell:
args = args + ['--shell', shell]
if gid:
args = args + ['--gid', gid]
if home:
args = args + ['--home', home]
if groups:
args = args + ['--group', groups]
if deleteFirst:
self.userdel(username)
args.append(username)
onlu.execute(args,
chroot=self.chroot,
ex=OnlRfsError("Adding user '%s' failed." % username))
if password is None:
onlu.execute(
"passwd -d %s" % username,
chroot=self.chroot,
ex=OnlRfsError("Error deleting password for user '%s'" %
username))
logger.info("user %s password %s", username, password)
if sudo:
sudoer = os.path.join(self.chroot, 'etc/sudoers.d', username)
self.chmod("777", os.path.dirname(sudoer))
with open(sudoer, "w") as f:
f.write("%s ALL=(ALL:ALL) NOPASSWD:ALL\n" % username)
self.chmod("0440", sudoer)
self.chown(sudoer, "root:root")
self.chmod("755", os.path.dirname(sudoer))
def user_password_set(self, username, password):
logger.info("user %s password now %s", username, password)
epassword = crypt.crypt(password, "$1$%s$" % self.gen_salt())
onlu.execute(
["usermod", "-p", epassword, username],
chroot=self.chroot,
ex=OnlRfsError("Error setting password for user '%s'" % username),
)
def useradd(
self,
username,
uid=None,
gid=None,
password=None,
shell="/bin/bash",
home=None,
groups=None,
sudo=False,
deleteFirst=True,
):
args = ["useradd", "--create-home"]
if uid:
args = args + ["--non-unique", "--uid", str(uid)]
if password:
epassword = crypt.crypt(password, "$1$%s$" % self.gen_salt())
args = args + ["-p", epassword]
if shell:
args = args + ["--shell", shell]
if gid:
args = args + ["--gid", gid]
if home:
args = args + ["--home", home]
if groups:
args = args + ["--group", groups]
if deleteFirst:
self.userdel(username)
args.append(username)
onlu.execute(args, chroot=self.chroot, ex=OnlRfsError("Adding user '%s' failed." % username))
if password is None:
onlu.execute(
"passwd -d %s" % username,
chroot=self.chroot,
ex=OnlRfsError("Error deleting password for user '%s'" % username),
)
logger.info("user %s password %s", username, password)
if sudo:
sudoer = os.path.join(self.chroot, "etc/sudoers.d", username)
self.chmod("777", os.path.dirname(sudoer))
with open(sudoer, "w") as f:
f.write("%s ALL=(ALL:ALL) NOPASSWD:ALL\n" % username)
self.chmod("0440", sudoer)
self.chown(sudoer, "root:root")
self.chmod("755", os.path.dirname(sudoer))
def update(self, dir_, packages):
ONLPM = "%s/tools/onlpm.py" % os.getenv('ONL')
with OnlRfsContext(dir_):
for pspec in packages:
for pkg in pspec.split(','):
logger.info("updating %s into %s", pkg, dir_)
cmd = (ONLPM, '--verbose',
'--sudo',
'--extract-dir', pkg, dir_,)
onlu.execute(cmd,
ex=OnlRfsError("update of %s failed" % pkg))
def update(self, dir_, packages):
ONLPM = "%s/tools/onlpm.py" % os.getenv('ONL')
with OnlRfsContext(dir_):
for pspec in packages:
for pkg in pspec.split(','):
logger.info("updating %s into %s", pkg, dir_)
cmd = (ONLPM, '--verbose',
'--sudo',
'--extract-dir', pkg, dir_,)
onlu.execute(cmd,
ex=OnlRfsError("update of %s failed" % pkg))
def dpkg_configure(self, dir_):
if self.arch == 'powerpc':
onlu.execute('sudo cp %s %s' %
(self.QEMU_PPC, os.path.join(dir_, 'usr/bin')))
if self.arch == 'armel':
onlu.execute('sudo cp %s %s' %
(self.QEMU_ARM, os.path.join(dir_, 'usr/bin')))
if self.arch == 'arm64':
onlu.execute('sudo cp %s %s' %
(self.QEMU_ARM64, os.path.join(dir_, 'usr/bin')))
onlu.execute('sudo cp %s %s' %
(os.path.join(os.getenv('ONL'), 'tools', 'scripts',
'base-files.postinst'),
os.path.join(dir_, 'var', 'lib', 'dpkg', 'info',
'base-files.postinst')))
script = os.path.join(dir_, "tmp/configure.sh")
with open(script, "w") as f:
os.chmod(script, 0700)
f.write("""
#!/bin/bash -ex
/bin/echo -e "#!/bin/sh\\nexit 101" >/usr/sbin/policy-rc.d
chmod +x /usr/sbin/policy-rc.d
export DEBIAN_FRONTEND=noninteractive
export DEBCONF_NONINTERACTIVE_SEEN=true
echo "127.0.0.1 localhost" >/etc/hosts
touch /etc/fstab
echo "localhost" >/etc/hostname
if [ -f /var/lib/dpkg/info/dash.preinst ]; then
/var/lib/dpkg/info/dash.preinst install
fi
if [ -f /usr/sbin/locale-gen ]; then
echo "en_US.UTF-8 UTF-8" >/etc/locale.gen
/usr/sbin/locale-gen
update-locale LANG=en_US.UTF-8
fi
dpkg --configure -a || true
dpkg --configure -a # configure any packages that failed the first time and abort on failure.
rm -f /usr/sbin/policy-rc.d
""")
logger.info("dpkg-configure filesystem...")
onlu.execute("sudo chroot %s /tmp/configure.sh" % dir_,
ex=OnlRfsError("Post Configuration failed."))
os.unlink(script)
def gmake_locked(self, target, operation):
buildpaths = []
if self._pkgs.get('build', True) and not os.environ.get('NOBUILD', False):
buildpaths = [
os.path.join(self._pkgs['__directory'], 'builds'),
os.path.join(self._pkgs['__directory'], 'BUILDS'),
]
for bp in buildpaths:
if os.path.exists(bp):
MAKE = os.environ.get('MAKE', "make")
cmd = MAKE + ' -C ' + bp + " " + os.environ.get('ONLPM_MAKE_OPTIONS', "") + " " + os.environ.get('ONL_MAKE_PARALLEL', "") + " " + target
onlu.execute(cmd,
ex=OnlPackageError('%s failed.' % operation))
def gmake_locked(self, target, operation):
buildpaths = []
if self._pkgs.get('build', True) and not os.environ.get('NOBUILD', False):
buildpaths = [
os.path.join(self._pkgs['__directory'], 'builds'),
os.path.join(self._pkgs['__directory'], 'BUILDS'),
]
for bp in buildpaths:
if os.path.exists(bp):
MAKE = os.environ.get('MAKE', "make")
V = " V=1 " if logger.level < logging.INFO else ""
cmd = MAKE + V + ' -C ' + bp + " " + os.environ.get('ONLPM_MAKE_OPTIONS', "") + " " + os.environ.get('ONL_MAKE_PARALLEL', "") + " " + target
onlu.execute(cmd,
ex=OnlPackageError('%s failed.' % operation))
def multistrap(self, dir_):
msconfig = self.ms.generate_file()
# Optional local package updates
for r in self.ms.localrepos:
logger.info("Updating %s" % r)
if os.path.exists(os.path.join(r, "Makefile")):
onlu.execute("make -C %s" % r)
if os.path.exists(dir_):
onlu.execute("sudo rm -rf %s" % dir_, ex=OnlRfsError("Could not remove target directory."))
if onlu.execute("sudo %s -d %s -f %s" % (self.MULTISTRAP, dir_, msconfig)) == 100:
raise OnlRfsError("Multistrap APT failure.")
if os.getenv("MULTISTRAP_DEBUG"):
raise OnlRfsError("Multistrap debug.")
def multistrap(self, dir_):
msconfig = self.ms.generate_file()
# Optional local package updates
for r in self.ms.localrepos:
logger.info("Updating %s" % r)
if os.path.exists(os.path.join(r, 'Makefile')):
onlu.execute("make -C %s" % r)
if os.path.exists(dir_):
onlu.execute("sudo rm -rf %s" % dir_,
ex=OnlRfsError("Could not remove target directory."))
if onlu.execute("sudo %s -d %s -f %s" % (self.MULTISTRAP, dir_, msconfig)) == 100:
raise OnlRfsError("Multistrap APT failure.")
if os.getenv("MULTISTRAP_DEBUG"):
raise OnlRfsError("Multistrap debug.")
def __enter__(self):
onlu.execute("sudo mount -t devtmpfs dev %s" % self.dev,
ex=OnlRfsError("Could not mount dev in rfs."))
onlu.execute("sudo mount -t proc proc %s" % self.proc,
ex=OnlRfsError("Could not mount proc in rfs."))
if os.path.islink(self.resolv) or os.path.exists(self.resolv):
onlu.execute("sudo mv %s %s" % (self.resolv, self.resolvb),
ex=OnlRfsError("Could not backup %s" % (self.resolv)))
onlu.execute("sudo cp /etc/resolv.conf %s" % (self.resolv),
ex=OnlRfsError("Could not copy resolv.conf"))
return self
def dpkg_configure(self, dir_):
if self.arch == 'powerpc':
onlu.execute('sudo cp %s %s' % (self.QEMU_PPC, os.path.join(dir_, 'usr/bin')))
if self.arch in [ 'armel', 'armhf' ]:
onlu.execute('sudo cp %s %s' % (self.QEMU_ARM, os.path.join(dir_, 'usr/bin')))
if self.arch == 'arm64':
onlu.execute('sudo cp %s %s' % (self.QEMU_ARM64, os.path.join(dir_, 'usr/bin')))
onlu.execute('sudo cp %s %s' % (os.path.join(os.getenv('ONL'), 'tools', 'scripts', 'base-files.postinst'),
os.path.join(dir_, 'var', 'lib', 'dpkg', 'info', 'base-files.postinst')));
script = os.path.join(dir_, "tmp/configure.sh")
with open(script, "w") as f:
os.chmod(script, 0700)
f.write("""#!/bin/bash -ex
/bin/echo -e "#!/bin/sh\\nexit 101" >/usr/sbin/policy-rc.d
chmod +x /usr/sbin/policy-rc.d
export DEBIAN_FRONTEND=noninteractive
export DEBCONF_NONINTERACTIVE_SEEN=true
echo "127.0.0.1 localhost" >/etc/hosts
touch /etc/fstab
echo "localhost" >/etc/hostname
if [ -f /var/lib/dpkg/info/dash.preinst ]; then
/var/lib/dpkg/info/dash.preinst install
fi
if [ -f /usr/sbin/locale-gen ]; then
echo "en_US.UTF-8 UTF-8" >/etc/locale.gen
/usr/sbin/locale-gen
update-locale LANG=en_US.UTF-8
fi
dpkg --configure -a || true
dpkg --configure -a # configure any packages that failed the first time and abort on failure.
rm -f /usr/sbin/policy-rc.d
""")
logger.info("dpkg-configure filesystem...")
onlu.execute("sudo chroot %s /tmp/configure.sh" % dir_,
ex=OnlRfsError("Post Configuration failed."))
os.unlink(script)
def groupadd(self, group, gid=None, unique=True, system=False, force=False, password=None):
args = [ 'groupadd' ]
if force:
args.append("--force")
if system:
args.append("--system")
if not unique:
args.append("--non-unique")
if password:
args = args + [ '--password', password ]
if gid:
args = args + [ '--gid', str(gid) ]
args.append(group)
onlu.execute(args,
chroot=self.chroot,
ex=OnlRfsError("Adding group '%s' failed." % group))
logger.info("added group %s", group)
def useradd(self, username, uid, password, shell, deleteFirst=True):
args = [ 'useradd', '--non-unique', '--shell', shell, '--home-dir', '/root',
'--uid', '0', '--gid', '0', '--group', 'root' ]
if deleteFirst:
self.userdel(username)
if password:
epassword=crypt.crypt(password, '$1$%s$' % self.gen_salt());
args = args + ['-p', epassword ]
args.append(username)
onlu.execute(args,
ex=OnlRfsError("Adding user '%s' failed." % username))
if password is None:
onlu.execute("passwd -d %s" % username,
chroot=self.chroot,
ex=OnlRfsError("Error deleting password for user '%s'" % username))
logger.info("user %s password %s", username, password)
def __exit__(self, eType, eValue, eTrace):
onlu.execute("sudo umount -l %s %s" % (self.dev, self.proc),
ex=OnlRfsError("Could not unmount dev and proc"))
onlu.execute("sudo rm %s" % (self.resolv),
ex=OnlRfsError("Could not remove resolv.conf"))
if os.path.islink(self.resolvb) or os.path.exists(self.resolvb):
onlu.execute(
"sudo mv %s %s" % (self.resolvb, self.resolv),
ex=OnlRfsError("Could not restore the resolv.conf backup"))
def __exit__(self, eType, eValue, eTrace):
onlu.execute("sudo umount -l %s %s" % (self.dev, self.proc),
ex=OnlRfsError("Could not unmount dev and proc"))
if self.rc:
onlu.execute("sudo rm %s" % (self.resolvconf),
ex=OnlRfsError("Could not remove new resolv.conf"))
if self.exists(self.resolvconfb):
onlu.execute("sudo mv %s %s" % (self.resolvconfb, self.resolvconf),
ex=OnlRfsError("Could not restore resolv.conf"))
def __exit__(self, eType, eValue, eTrace):
onlu.execute("sudo umount -l %s %s" % (self.dev, self.proc),
ex=OnlRfsError("Could not unmount dev and proc"))
if self.rc:
onlu.execute("sudo rm %s" % (self.resolvconf),
ex=OnlRfsError("Could not remove new resolv.conf"))
if self.exists(self.resolvconfb):
onlu.execute("sudo mv %s %s" % (self.resolvconfb, self.resolvconf),
ex=OnlRfsError("Could not restore resolv.conf"))
def __enter__(self):
try:
onlu.execute("sudo mount -t devtmpfs dev %s" % self.dev,
ex=OnlRfsError("Could not mount dev in rfs."))
onlu.execute("sudo mount -t proc proc %s" % self.proc,
ex=OnlRfsError("Could not mount proc in rfs."))
if self.rc:
if self.exists(self.resolvconf):
onlu.execute("sudo mv %s %s" % (self.resolvconf, self.resolvconfb),
ex=OnlRfsError("Could not backup resolv.conf"))
onlu.execute("sudo cp --remove-destination /etc/resolv.conf %s" % (self.resolvconf),
ex=OnlRfsError("Could install new resolv.conf"))
return self
except Exception, e:
logger.error("Exception %s in OnlRfsContext::__enter__" % e)
self.__exit__(None, None, None)
raise e
def __enter__(self):
try:
onlu.execute("sudo mount -t devtmpfs dev %s" % self.dev,
ex=OnlRfsError("Could not mount dev in rfs."))
onlu.execute("sudo mount -t proc proc %s" % self.proc,
ex=OnlRfsError("Could not mount proc in rfs."))
if self.rc:
if self.exists(self.resolvconf):
onlu.execute("sudo mv %s %s" % (self.resolvconf, self.resolvconfb),
ex=OnlRfsError("Could not backup resolv.conf"))
onlu.execute("sudo cp --remove-destination /etc/resolv.conf %s" % (self.resolvconf),
ex=OnlRfsError("Could install new resolv.conf"))
return self
except Exception, e:
logger.error("Exception %s in OnlRfsContext::__enter__" % e)
self.__exit__(None, None, None)
raise e
def extract(self, pkg, dstdir=None, prefix=True, force=False, remove_ts=False, sudo=False):
"""Extract the given package.
pkg : The package identifier. Must be unique in the repo.
dstdir : The parent directory which will contain the extracted package contents.
The default is the local repo's extract cache.
force: If True, the package will be extracted even if its contents are already valid in the extract cache."""
PKG_TIMESTAMP = '.PKG.TIMESTAMP'
with self.lock:
path = self.lookup(pkg)
if path:
if dstdir is None:
dstdir = self.extracts
if prefix:
edir = os.path.join(dstdir, pkg.replace(':', '_'))
else:
edir = dstdir
if not force and os.path.exists(os.path.join(edir, PKG_TIMESTAMP)):
if (os.path.getmtime(os.path.join(edir, PKG_TIMESTAMP)) ==
os.path.getmtime(path)):
# Existing extract is identical to source package
logger.debug("Existing extract for %s matches the package file." % pkg)
else:
# Existing extract must be removed.
logger.info("Existing extract for %s does not match." % pkg)
force=True
else:
# Status unknown. Really shouldn't happen.
force=True
if force:
if os.path.exists(edir) and prefix:
shutil.rmtree(edir)
if not os.path.exists(edir):
os.makedirs(edir)
onlu.execute([ 'dpkg', '-x', path, edir ], sudo=sudo)
onlu.execute([ 'touch', '-r', path, os.path.join(edir, PKG_TIMESTAMP) ], sudo=sudo)
if remove_ts and os.path.exists(os.path.join(edir, PKG_TIMESTAMP)):
onlu.execute([ 'rm', os.path.join(edir, PKG_TIMESTAMP) ], sudo=sudo)
return edir
return False
def extract(self, pkg, dstdir=None, prefix=True, force=False, remove_ts=False, sudo=False):
"""Extract the given package.
pkg : The package identifier. Must be unique in the repo.
dstdir : The parent directory which will contain the extracted package contents.
The default is the local repo's extract cache.
force: If True, the package will be extracted even if its contents are already valid in the extract cache."""
PKG_TIMESTAMP = '.PKG.TIMESTAMP'
with self.lock:
path = self.lookup(pkg)
if path:
if dstdir is None:
dstdir = self.extracts
if prefix:
edir = os.path.join(dstdir, pkg.replace(':', '_'))
else:
edir = dstdir
if not force and os.path.exists(os.path.join(edir, PKG_TIMESTAMP)):
if (os.path.getmtime(os.path.join(edir, PKG_TIMESTAMP)) ==
os.path.getmtime(path)):
# Existing extract is identical to source package
logger.debug("Existing extract for %s matches the package file." % pkg)
else:
# Existing extract must be removed.
logger.warn("Existing extract for %s does not match." % pkg)
force=True
else:
# Status unknown. Really shouldn't happen.
force=True
if force:
if os.path.exists(edir) and prefix:
shutil.rmtree(edir)
if not os.path.exists(edir):
os.makedirs(edir)
onlu.execute([ 'dpkg', '-x', path, edir ], sudo=sudo)
onlu.execute([ 'touch', '-r', path, os.path.join(edir, PKG_TIMESTAMP) ], sudo=sudo)
if remove_ts and os.path.exists(os.path.join(edir, PKG_TIMESTAMP)):
onlu.execute([ 'rm', os.path.join(edir, PKG_TIMESTAMP) ], sudo=sudo)
return edir
return False
def configure(self, dir_):
try:
onlu.execute(
"sudo mount -t devtmpfs dev %s" % os.path.join(dir_, "dev"),
ex=OnlRfsError("Could not mount dev in new filesystem."),
)
onlu.execute(
"sudo mount -t proc proc %s" % os.path.join(dir_, "proc"),
ex=OnlRfsError("Could not mount proc in new filesystem."),
)
script = os.path.join(dir_, "tmp/configure.sh")
if not os.getenv("NO_DPKG_CONFIGURE"):
self.dpkg_configure(dir_)
Configure = self.config.get("Configure", None)
if Configure:
for overlay in Configure.get("overlays", []):
logger.info("Overlay %s..." % overlay)
onlu.execute(
'tar -C %s -c --exclude "*~" . | sudo tar -C %s -x -v --no-same-owner' % (overlay, dir_),
ex=OnlRfsError("Overlay '%s' failed." % overlay),
)
for update in Configure.get("update-rc.d", []):
onlu.execute(
"sudo chroot %s /usr/sbin/update-rc.d %s" % (dir_, update),
ex=OnlRfsError("update-rc.d %s failed." % (update)),
)
for script in Configure.get("scripts", []):
logger.info("Configuration script %s..." % script)
onlu.execute("sudo %s %s" % (script, dir_), ex=OnlRfsError("script '%s' failed." % script))
for command in Configure.get("commands", []):
if "__rfs__" in command:
command = command % dict(__rfs__=dir_)
logger.info("Configuration command '%s'..." % command)
onlu.execute(command, ex=OnlRfsError("Command '%s' failed." % command))
for (user, values) in Configure.get("users", {}).iteritems():
ua = OnlRfsSystemAdmin(dir_)
if user == "root":
if "password" in values:
ua.user_password_set(user, values["password"])
else:
ua.useradd(username=user, **values)
options = Configure.get("options", {})
if options.get("clean", False):
logger.info("Cleaning Filesystem...")
onlu.execute("sudo chroot %s /usr/bin/apt-get clean" % dir_)
onlu.execute("sudo chroot %s /usr/sbin/localepurge" % dir_)
onlu.execute("sudo chroot %s find /usr/share/doc -type f -delete" % dir_)
onlu.execute("sudo chroot %s find /usr/share/man -type f -delete" % dir_)
if not options.get("securetty", True):
f = os.path.join(dir_, "etc/securetty")
if os.path.exists(f):
logger.info("Removing %s" % f)
onlu.execute("sudo rm %s" % f, ex=OnlRfsError("Could not remove file %s" % f))
if not options.get("ttys", False):
f = os.path.join(dir_, "etc/inittab")
ua.chmod("a+w", f)
ua.chmod("a+w", os.path.dirname(f))
logger.info("Clearing %s ttys..." % f)
for line in fileinput.input(f, inplace=True):
if re.match("^[123456]:.*", line):
line = "#" + line
print line,
ua.chmod("go-w", f)
ua.chmod("go-w", os.path.dirname(f))
if options.get("console", True):
logger.info("Configuring Console Access in %s" % f)
f = os.path.join(dir_, "etc/inittab")
ua.chmod("a+w", f)
ua.chmod("a+w", os.path.dirname(f))
with open(f, "a") as h:
h.write("T0:23:respawn:/sbin/pgetty\n")
ua.chmod("go-w", f)
ua.chmod("go-w", os.path.dirname(f))
for (mf, fields) in Configure.get("manifests", {}).iteritems():
logger.info("Configuring manifest %s..." % mf)
if mf.startswith("/"):
mf = mf[1:]
mname = os.path.join(dir_, mf)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(mname))
onlu.execute("sudo touch %s" % mname)
onlu.execute("sudo chmod a+w %s" % mname)
md = {}
md["version"] = json.load(open(fields["version"]))
md["arch"] = self.arch
if os.path.exists(fields["platforms"]):
md["platforms"] = yaml.load(open(fields["platforms"]))
else:
md["platforms"] = fields["platforms"].split(",")
with open(mname, "w") as f:
json.dump(md, f, indent=2)
onlu.execute("sudo chmod a-w %s" % mname)
for (fname, v) in Configure.get("files", {}).get("add", {}).iteritems():
if fname.startswith("/"):
fname = fname[1:]
dst = os.path.join(dir_, fname)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(dst))
onlu.execute("sudo touch %s" % dst)
onlu.execute("sudo chmod a+w %s" % dst)
if os.path.exists(v):
shutil.copy(v, dst)
else:
with open(dst, "w") as f:
f.write("%s\n" % v)
for fname in Configure.get("files", {}).get("remove", []):
if fname.startswith("/"):
fname = fname[1:]
f = os.path.join(dir_, fname)
if os.path.exists(f):
onlu.execute("sudo rm -rf %s" % f)
if Configure.get("issue"):
issue = Configure.get("issue")
fn = os.path.join(dir_, "etc/issue")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
fn = os.path.join(dir_, "etc/issue.net")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
finally:
onlu.execute("sudo umount -l %s %s" % (os.path.join(dir_, "dev"), os.path.join(dir_, "proc")))
def chown(file_, ownspec):
onlu.execute(
"sudo chown %s %s" % (ownspec, file_),
ex=OnlRfsError("Could not change ownership (%s) on file %s" %
(ownspec, file_)))
def chmod(mode, file_):
onlu.execute(
"sudo chmod %s %s" % (mode, file_),
ex=OnlRfsError("Could not change permissions (%s) on file %s" %
(mode, file_)))
def configure(self, dir_):
if not os.getenv('NO_DPKG_CONFIGURE'):
with OnlRfsContext(dir_, resolvconf=False):
self.dpkg_configure(dir_)
with OnlRfsContext(dir_):
os_release = os.path.join(dir_, 'etc', 'os-release')
if os.path.exists(os_release):
# Convert /etc/os-release to /etc/os-release.json
import shlex
contents = open(os_release).read()
d = dict(token.split('=') for token in shlex.split(contents))
ua = OnlRfsSystemAdmin(dir_)
ua.chmod('a+rwx', os.path.dirname(os_release))
with open(
os.path.join(os.path.dirname(os_release),
'os-release.json'), "w") as f:
f.write(json.dumps(d))
ua.chmod('0755', os.path.dirname(os_release))
Configure = self.config.get('Configure', None)
if Configure:
for cmd in Configure.get('run', []):
onlu.execute("sudo chroot %s %s" % (dir_, cmd),
ex=OnlRfsError("run command '%s' failed" %
cmd))
for overlay in Configure.get('overlays', []):
logger.info("Overlay %s..." % overlay)
onlu.execute(
'tar -C %s -c --exclude "*~" . | sudo tar -C %s -x -v --no-same-owner'
% (overlay, dir_),
ex=OnlRfsError("Overlay '%s' failed." % overlay))
for update in Configure.get('update-rc.d', []):
onlu.execute("sudo chroot %s /usr/sbin/update-rc.d %s" %
(dir_, update),
ex=OnlRfsError("update-rc.d %s failed." %
(update)))
for script in Configure.get('scripts', []):
logger.info("Configuration script %s..." % script)
onlu.execute("sudo %s %s" % (script, dir_),
ex=OnlRfsError("script '%s' failed." %
script))
for command in Configure.get('commands', []):
if '__rfs__' in command:
command = command % dict(__rfs__=dir_)
logger.info("Configuration command '%s'..." % command)
onlu.execute(command,
ex=OnlRfsError("Command '%s' failed." %
command))
ua = OnlRfsSystemAdmin(dir_)
for (group, values) in Configure.get('groups', {}).iteritems():
ua.groupadd(group=group, **values if values else {})
for (user, values) in Configure.get('users', {}).iteritems():
if user == 'root':
if 'password' in values:
ua.user_password_set(user, values['password'])
else:
ua.useradd(username=user, **values)
options = Configure.get('options', {})
if options.get('clean', False):
logger.info("Cleaning Filesystem...")
onlu.execute('sudo chroot %s /usr/bin/apt-get clean' %
dir_)
onlu.execute('sudo chroot %s /usr/sbin/localepurge' % dir_)
onlu.execute(
'sudo chroot %s find /usr/share/doc -type f -not -name asr.json -delete'
% dir_)
onlu.execute(
'sudo chroot %s find /usr/share/man -type f -delete' %
dir_)
if 'PermitRootLogin' in options:
config = os.path.join(dir_, 'etc/ssh/sshd_config')
ua.chmod('a+rw', config)
lines = open(config).readlines()
with open(config, "w") as f:
for line in lines:
if line.startswith('PermitRootLogin'):
v = options['PermitRootLogin']
logger.info("Setting PermitRootLogin to %s" %
v)
f.write('PermitRootLogin %s\n' % v)
else:
f.write(line)
ua.chmod('644', config)
if not options.get('securetty', True):
f = os.path.join(dir_, 'etc/securetty')
if os.path.exists(f):
logger.info("Removing %s" % f)
onlu.execute('sudo rm %s' % f,
ex=OnlRfsError(
'Could not remove file %s' % f))
if not options.get('ttys', False):
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
logger.info("Clearing %s ttys..." % f)
for line in fileinput.input(f, inplace=True):
if re.match("^[123456]:.*", line):
line = "#" + line
print line,
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('console', True):
logger.info('Configuring Console Access in %s' % f)
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
with open(f, 'a') as h:
h.write("T0:23:respawn:/sbin/pgetty\n")
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('asr', None):
asropts = options.get('asr')
logger.info("Gathering ASR documentation...")
sys.path.append("%s/sm/infra/tools" % os.getenv('ONL'))
import asr
asro = asr.AimSyslogReference()
asro.merge(dir_)
asro.format(os.path.join(dir_, asropts['file']),
fmt=asropts['format'])
for (mf, fields) in Configure.get('manifests', {}).iteritems():
logger.info("Configuring manifest %s..." % mf)
if mf.startswith('/'):
mf = mf[1:]
mname = os.path.join(dir_, mf)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(mname))
onlu.execute("sudo touch %s" % mname)
onlu.execute("sudo chmod a+w %s" % mname)
md = {}
md['version'] = json.load(open(fields['version']))
md['arch'] = self.arch
if os.path.exists(fields['platforms']):
md['platforms'] = yaml.load(open(fields['platforms']))
else:
md['platforms'] = fields['platforms'].split(',')
for (k, v) in fields.get('keys', {}).iteritems():
if k in md:
md[k].update(v)
else:
md[k] = v
with open(mname, "w") as f:
json.dump(md, f, indent=2)
onlu.execute("sudo chmod a-w %s" % mname)
for (fname, v) in Configure.get('files',
{}).get('add', {}).iteritems():
if fname.startswith('/'):
fname = fname[1:]
dst = os.path.join(dir_, fname)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(dst))
onlu.execute("sudo touch %s" % dst)
onlu.execute("sudo chmod a+w %s" % dst)
if os.path.exists(v):
shutil.copy(v, dst)
else:
with open(dst, "w") as f:
f.write("%s\n" % v)
for fname in Configure.get('files', {}).get('remove', []):
if fname.startswith('/'):
fname = fname[1:]
f = os.path.join(dir_, fname)
if os.path.exists(f):
onlu.execute("sudo rm -rf %s" % f)
if Configure.get('issue'):
issue = Configure.get('issue')
fn = os.path.join(dir_, "etc/issue")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
fn = os.path.join(dir_, "etc/issue.net")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
def user_shell_set(self, username, shell):
onlu.execute('chsh --shell %s %s' % (shell, username),
chroot=self.chroot,
ex=OnlRfsError("Error setting shell %s for user %s" %
(shell, username)))
def configure(self, dir_):
try:
onlu.execute("sudo mount -t devtmpfs dev %s" % os.path.join(dir_, "dev"),
ex=OnlRfsError("Could not mount dev in new filesystem."))
onlu.execute("sudo mount -t proc proc %s" % os.path.join(dir_, "proc"),
ex=OnlRfsError("Could not mount proc in new filesystem."))
script = os.path.join(dir_, "tmp/configure.sh")
if not os.getenv('NO_DPKG_CONFIGURE'):
self.dpkg_configure(dir_)
Configure = self.config.get('Configure', None)
if Configure:
for overlay in Configure.get('overlays', []):
logger.info("Overlay %s..." % overlay)
onlu.execute('tar -C %s -c --exclude "*~" . | sudo tar -C %s -x -v --no-same-owner' % (overlay, dir_),
ex=OnlRfsError("Overlay '%s' failed." % overlay))
for update in Configure.get('update-rc.d', []):
onlu.execute("sudo chroot %s /usr/sbin/update-rc.d %s" % (dir_, update),
ex=OnlRfsError("update-rc.d %s failed." % (update)))
for script in Configure.get('scripts', []):
logger.info("Configuration script %s..." % script)
onlu.execute("sudo %s %s" % (script, dir_),
ex=OnlRfsError("script '%s' failed." % script))
for command in Configure.get('commands', []):
if '__rfs__' in command:
command = command % dict(__rfs__=dir_)
logger.info("Configuration command '%s'..." % command)
onlu.execute(command,
ex=OnlRfsError("Command '%s' failed." % command))
for (user, values) in Configure.get('users', {}).iteritems():
ua = OnlRfsSystemAdmin(dir_)
if 'password' in values:
ua.user_password_set(user, values['password'])
options = Configure.get('options', {})
if options.get('clean', False):
logger.info("Cleaning Filesystem...")
if not options.get('securetty', True):
f = os.path.join(dir_, 'etc/securetty')
if os.path.exists(f):
logger.info("Removing %s" % f)
onlu.execute('sudo rm %s' % f,
ex=OnlRfsError('Could not remove file %s' % f))
if not options.get('ttys', False):
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
logger.info("Clearing %s ttys..." % f)
for line in fileinput.input(f, inplace=True):
if re.match("^[123456]:.*", line):
line = "#" + line
print line,
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('console', True):
logger.info('Configuring Console Access in %s' % f)
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
with open(f, 'a') as h:
h.write("T0:23:respawn:/sbin/pgetty\n")
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
manifest = Configure.get('manifest', {})
if manifest:
mname = "%s/etc/onl/rootfs/manifest.json" % dir_
onlu.execute("sudo mkdir -p %s" % os.path.dirname(mname))
onlu.execute("sudo touch %s" % mname)
onlu.execute("sudo chmod a+w %s" % mname)
md = {}
md['version'] = json.load(open(manifest['version']))
md['arch'] = self.arch
if os.path.exists(manifest['platforms']):
md['platforms'] = yaml.load(open(manifest['platforms']))
else:
md['platforms'] = manifest['platforms'].split(',')
with open(mname, "w") as f:
json.dump(md, f, indent=2)
onlu.execute("sudo chmod a-w %s" % mname)
finally:
onlu.execute("sudo umount -l %s %s" % (os.path.join(dir_, "dev"), os.path.join(dir_, "proc")))
def build(self, dir_=None):
"""Build the debian package.
When this method is called it is assumed that all file
prerequisites for the package have already been built
or met. This is simply the packaging stage.
'dir_' : This is the output directory in which the package
should be left. If unspecified the package's local directory
will contain the package file."""
if 'external' in self.pkg:
# Package file built externally
epkg = self.pkg['external']
if os.path.exists(epkg):
return epkg
else:
raise OnlPackageError("The external package file '%s' does not exist." % epkg);
# Make sure all required files exist
if 'files' in self.pkg:
self._validate_files('files', True)
if 'optional-files' in self.pkg:
self._validate_files('optional-files', False)
# If dir_ is not specified, leave package in local package directory.
if dir_ is None:
dir_ = self.dir
workdir = tempfile.mkdtemp()
root = os.path.join(workdir, "root");
os.mkdir(root);
# The package file will be built into the workdir
self.pkg['__workdir'] = workdir
for (src,dst) in self.pkg.get('files', {}):
OnlPackage.copyf(src, dst, root)
for (src,dst) in self.pkg.get('optional-files', {}):
if os.path.exists(src):
OnlPackage.copyf(src, dst, root)
for (link,src) in self.pkg.get('links', {}).iteritems():
logger.info("Linking %s -> %s..." % (link, src))
link = os.path.join(root, link)
os.symlink(src, link)
#
# FPM doesn't seem to have a doc option so we copy documentation
# files directly into place.
#
docpath = os.path.join(root, "usr/share/doc/%(name)s" % self.pkg)
if not os.path.exists(docpath):
os.makedirs(docpath)
for src in self.pkg.get('docs', []):
if not os.path.exists(src):
raise OnlPackageError("Documentation source file '%s' does not exist." % src)
shutil.copy(src, docpath)
changelog = os.path.join(workdir, 'changelog')
copyright_ = os.path.join(workdir, 'copyright')
#
# Export changelog and copyright files from the PKG dict
# to the workdir for processing.
#
# The copyright and changelog data can be embedded directly
# int the PKG file or kept as separate files.
#
def copy_str_or_file(src, dst):
if os.path.exists(src):
shutil.copyfile(src, dst)
else:
with open(dst, "w") as f:
f.write(src)
f.write("\n")
copy_str_or_file(self.pkg['copyright'], copyright_)
copy_str_or_file(self.pkg['changelog'], changelog)
############################################################
#
# Invoke fpm with all necessary options.
#
############################################################
self.pkg['__root'] = root
command = """fpm -p %(__workdir)s -f -C %(__root)s -s dir -t deb -n %(name)s -v %(version)s -a %(arch)s -m %(maintainer)s --description "%(description)s" --url "%(url)s" --license "%(license)s" --vendor "%(vendor)s" """ % self.pkg
for dep in self.pkg.get('depends', []):
command = command + "-d %s " % dep
for provides in onlu.sflatten(self.pkg.get('provides', [])):
command = command + "--provides %s " % provides
for conflicts in onlu.sflatten(self.pkg.get('conflicts', [])):
command = command + "--conflicts %s " % conflicts
for replaces in onlu.sflatten(self.pkg.get('replaces', [])):
command = command + "--replaces %s " % replaces
if 'virtual' in self.pkg:
command = command + "--provides %(v)s --conflicts %(v)s --replaces %(v)s " % dict(v=self.pkg['virtual'])
if 'priority' in self.pkg:
command = command + "--deb-priority %s " % self.pkg['priority']
if 'init' in self.pkg:
if not os.path.exists(self.pkg['init']):
raise OnlPackageError("Init script '%s' does not exist." % self.pkg['init'])
command = command + "--deb-init %s " % self.pkg['init']
if self.pkg.get('init-after-install', True):
command = command + "--after-install %s " % OnlPackageAfterInstallScript(self.pkg['init'], dir=workdir).name
if self.pkg.get('init-before-remove', True):
command = command + "--before-remove %s " % OnlPackageBeforeRemoveScript(self.pkg['init'], dir=workdir).name
if self.pkg.get('init-after-remove', True):
command = command + "--after-remove %s " % OnlPackageAfterRemoveScript(self.pkg['init'], dir=workdir).name
if self.pkg.get('asr', False):
with onlu.Profiler() as profiler:
# Generate the ASR documentation for this package.
sys.path.append("%s/sm/infra/tools" % os.getenv('ONL'))
import asr
asro = asr.AimSyslogReference()
asro.extract(workdir)
asro.format(os.path.join(docpath, asr.AimSyslogReference.ASR_NAME), 'json')
profiler.log("ASR generation for %(name)s" % self.pkg)
############################################################
if logger.level < logging.INFO:
command = command + "--verbose "
onlu.execute(command)
# Grab the package from the workdir. There can be only one.
files = glob.glob(os.path.join(workdir, '*.deb'))
if len(files) == 0:
raise OnlPackageError("No debian package.")
elif len(files) > 1:
raise OnlPackageError("Too many packages.")
else:
# Move to the target directory
shutil.copy(files[0], dir_)
# Remove entire work directory.
shutil.rmtree(workdir)
# Return the path to the final built package
return os.path.join(dir_, os.path.basename(files[0]))
def contents(self, pkg):
with self.lock:
path = self.lookup(pkg)
if path:
print "** %s contents:" % path
onlu.execute(['dpkg', '-c', path])
def build(self, dir_=None):
"""Build the debian package.
When this method is called it is assumed that all file
prerequisites for the package have already been built
or met. This is simply the packaging stage.
'dir_' : This is the output directory in which the package
should be left. If unspecified the package's local directory
will contain the package file."""
if 'external' in self.pkg:
# Package file built externally
epkg = self.pkg['external']
if os.path.exists(epkg):
return epkg
else:
raise OnlPackageError("The external package file '%s' does not exist." % epkg);
# Make sure all required files exist
if 'files' in self.pkg:
self._validate_files()
# If dir_ is not specified, leave package in local package directory.
if dir_ is None:
dir_ = self.dir
workdir = tempfile.mkdtemp()
root = os.path.join(workdir, "root");
os.mkdir(root);
# The package file will be built into the workdir
self.pkg['__workdir'] = workdir
for (src,dst) in self.pkg.get('files', {}):
if dst.startswith('/'):
dst = dst[1:]
if os.path.isdir(src):
#
# Copy entire src directory to target directory
#
dstpath = os.path.join(root, dst)
logger.debug("Copytree %s -> %s" % (src, dstpath))
shutil.copytree(src, dstpath)
else:
#
# If the destination ends in a '/' it means copy the filename
# as-is to that directory.
#
# If not, its a full rename to the destination.
#
if dst.endswith('/'):
dstpath = os.path.join(root, dst)
if not os.path.exists(dstpath):
os.makedirs(dstpath)
shutil.copy(src, dstpath)
else:
dstpath = os.path.join(root, os.path.dirname(dst))
if not os.path.exists(dstpath):
os.makedirs(dstpath)
shutil.copyfile(src, os.path.join(root, dst))
shutil.copymode(src, os.path.join(root, dst))
for (link,src) in self.pkg.get('links', {}).iteritems():
logger.info("Linking %s -> %s..." % (link, src))
link = os.path.join(root, link)
os.symlink(src, link)
#
# FPM doesn't seem to have a doc option so we copy documentation
# files directly into place.
#
for src in self.pkg.get('docs', []):
if not os.path.exists(src):
raise OnlPackageError("Documentation source file '%s' does not exist." % src)
dstpath = os.path.join(root, "usr/share/doc/%(name)s" % self.pkg)
if not os.path.exists(dstpath):
os.makedirs(dstpath)
shutil.copy(src, dstpath)
changelog = os.path.join(workdir, 'changelog')
copyright_ = os.path.join(workdir, 'copyright')
#
# Export changelog and copyright files from the PKG dict
# to the workdir for processing.
#
# The copyright and changelog data can be embedded directly
# int the PKG file or kept as separate files.
#
def copy_str_or_file(src, dst):
if os.path.exists(src):
shutil.copyfile(src, dst)
else:
with open(dst, "w") as f:
f.write(src)
f.write("\n")
copy_str_or_file(self.pkg['copyright'], copyright_)
copy_str_or_file(self.pkg['changelog'], changelog)
############################################################
#
# Invoke fpm with all necessary options.
#
############################################################
self.pkg['__root'] = root
command = """fpm -p %(__workdir)s -f -C %(__root)s -s dir -t deb -n %(name)s -v %(version)s -a %(arch)s -m %(maintainer)s --description "%(description)s" --url "%(url)s" --license "%(license)s" --vendor "%(vendor)s" """ % self.pkg
for dep in self.pkg.get('depends', []):
command = command + "-d %s " % dep
for provides in onlu.sflatten(self.pkg.get('provides', [])):
command = command + "--provides %s " % provides
if 'init' in self.pkg:
if not os.path.exists(self.pkg['init']):
raise OnlPackageError("Init script '%s' does not exist." % self.pkg['init'])
command = command + "--deb-init %s" % self.pkg['init']
if 'post-install' in self.pkg:
if not os.path.exists(self.pkg['post-install']):
raise OnlPackageError("Post-install script '%s' does not exist." % self.pkg['post-install'])
command = command + "--after-install %s" % self.pkg['post-install']
if logger.level < logging.INFO:
command = command + "--verbose "
onlu.execute(command)
# Grab the package from the workdir. There can be only one.
sys.stdout.write(workdir)
files = glob.glob(os.path.join(workdir, '*.deb'))
if len(files) == 0:
raise OnlPackageError("No debian package.")
elif len(files) > 1:
raise OnlPackageError("Too many packages.")
else:
# Move to the target directory
shutil.copy(files[0], dir_)
# Remove entire work directory.
shutil.rmtree(workdir)
# Return the path to the final built package
return os.path.join(dir_, os.path.basename(files[0]))
def contents(self, pkg):
with self.lock:
path = self.lookup(pkg)
if path:
print "** %s contents:" % path
onlu.execute(['dpkg', '-c', path])
def configure(self, dir_):
try:
onlu.execute(
"sudo mount -t devtmpfs dev %s" % os.path.join(dir_, "dev"),
ex=OnlRfsError("Could not mount dev in new filesystem."))
onlu.execute(
"sudo mount -t proc proc %s" % os.path.join(dir_, "proc"),
ex=OnlRfsError("Could not mount proc in new filesystem."))
script = os.path.join(dir_, "tmp/configure.sh")
if not os.getenv('NO_DPKG_CONFIGURE'):
self.dpkg_configure(dir_)
Configure = self.config.get('Configure', None)
if Configure:
for overlay in Configure.get('overlays', []):
logger.info("Overlay %s..." % overlay)
onlu.execute(
'tar -C %s -c --exclude "*~" . | sudo tar -C %s -x -v --no-same-owner'
% (overlay, dir_),
ex=OnlRfsError("Overlay '%s' failed." % overlay))
for update in Configure.get('update-rc.d', []):
onlu.execute("sudo chroot %s /usr/sbin/update-rc.d %s" %
(dir_, update),
ex=OnlRfsError("update-rc.d %s failed." %
(update)))
for script in Configure.get('scripts', []):
logger.info("Configuration script %s..." % script)
onlu.execute("sudo %s %s" % (script, dir_),
ex=OnlRfsError("script '%s' failed." %
script))
for command in Configure.get('commands', []):
if '__rfs__' in command:
command = command % dict(__rfs__=dir_)
logger.info("Configuration command '%s'..." % command)
onlu.execute(command,
ex=OnlRfsError("Command '%s' failed." %
command))
for (user, values) in Configure.get('users', {}).iteritems():
ua = OnlRfsSystemAdmin(dir_)
if 'password' in values:
ua.user_password_set(user, values['password'])
options = Configure.get('options', {})
if options.get('clean', False):
logger.info("Cleaning Filesystem...")
if not options.get('securetty', True):
f = os.path.join(dir_, 'etc/securetty')
if os.path.exists(f):
logger.info("Removing %s" % f)
onlu.execute('sudo rm %s' % f,
ex=OnlRfsError(
'Could not remove file %s' % f))
if not options.get('ttys', False):
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
logger.info("Clearing %s ttys..." % f)
for line in fileinput.input(f, inplace=True):
if re.match("^[123456]:.*", line):
line = "#" + line
print line,
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('console', True):
logger.info('Configuring Console Access in %s' % f)
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
with open(f, 'a') as h:
h.write("T0:23:respawn:/sbin/pgetty\n")
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
manifest = Configure.get('manifest', {})
if manifest:
mname = "%s/etc/onl/rootfs/manifest.json" % dir_
onlu.execute("sudo mkdir -p %s" % os.path.dirname(mname))
onlu.execute("sudo touch %s" % mname)
onlu.execute("sudo chmod a+w %s" % mname)
md = {}
md['version'] = json.load(open(manifest['version']))
md['arch'] = self.arch
if os.path.exists(manifest['platforms']):
md['platforms'] = yaml.load(open(
manifest['platforms']))
else:
md['platforms'] = manifest['platforms'].split(',')
with open(mname, "w") as f:
json.dump(md, f, indent=2)
onlu.execute("sudo chmod a-w %s" % mname)
finally:
onlu.execute(
"sudo umount -l %s %s" %
(os.path.join(dir_, "dev"), os.path.join(dir_, "proc")))
def build(self, dir_=None):
"""Build the debian package.
When this method is called it is assumed that all file
prerequisites for the package have already been built
or met. This is simply the packaging stage.
'dir_' : This is the output directory in which the package
should be left. If unspecified the package's local directory
will contain the package file."""
if 'external' in self.pkg:
# Package file built externally
epkg = self.pkg['external']
if os.path.exists(epkg):
return epkg
else:
raise OnlPackageError(
"The external package file '%s' does not exist." % epkg)
# Make sure all required files exist
if 'files' in self.pkg:
self._validate_files()
# If dir_ is not specified, leave package in local package directory.
if dir_ is None:
dir_ = self.dir
workdir = tempfile.mkdtemp()
root = os.path.join(workdir, "root")
os.mkdir(root)
# The package file will be built into the workdir
self.pkg['__workdir'] = workdir
for (src, dst) in self.pkg.get('files', {}):
if dst.startswith('/'):
dst = dst[1:]
if os.path.isdir(src):
#
# Copy entire src directory to target directory
#
dstpath = os.path.join(root, dst)
logger.debug("Copytree %s -> %s" % (src, dstpath))
shutil.copytree(src, dstpath)
else:
#
# If the destination ends in a '/' it means copy the filename
# as-is to that directory.
#
# If not, its a full rename to the destination.
#
if dst.endswith('/'):
dstpath = os.path.join(root, dst)
if not os.path.exists(dstpath):
os.makedirs(dstpath)
shutil.copy(src, dstpath)
else:
dstpath = os.path.join(root, os.path.dirname(dst))
if not os.path.exists(dstpath):
os.makedirs(dstpath)
shutil.copyfile(src, os.path.join(root, dst))
shutil.copymode(src, os.path.join(root, dst))
for (link, src) in self.pkg.get('links', {}).iteritems():
logger.info("Linking %s -> %s..." % (link, src))
link = os.path.join(root, link)
os.symlink(src, link)
#
# FPM doesn't seem to have a doc option so we copy documentation
# files directly into place.
#
for src in self.pkg.get('docs', []):
if not os.path.exists(src):
raise OnlPackageError(
"Documentation source file '%s' does not exist." % src)
dstpath = os.path.join(root, "usr/share/doc/%(name)s" % self.pkg)
if not os.path.exists(dstpath):
os.makedirs(dstpath)
shutil.copy(src, dstpath)
changelog = os.path.join(workdir, 'changelog')
copyright_ = os.path.join(workdir, 'copyright')
#
# Export changelog and copyright files from the PKG dict
# to the workdir for processing.
#
# The copyright and changelog data can be embedded directly
# int the PKG file or kept as separate files.
#
def copy_str_or_file(src, dst):
if os.path.exists(src):
shutil.copyfile(src, dst)
else:
with open(dst, "w") as f:
f.write(src)
f.write("\n")
copy_str_or_file(self.pkg['copyright'], copyright_)
copy_str_or_file(self.pkg['changelog'], changelog)
############################################################
#
# Invoke fpm with all necessary options.
#
############################################################
self.pkg['__root'] = root
command = """fpm -p %(__workdir)s -f -C %(__root)s -s dir -t deb -n %(name)s -v %(version)s -a %(arch)s -m %(maintainer)s --description "%(description)s" --url "%(url)s" --license "%(license)s" --vendor "%(vendor)s" """ % self.pkg
for dep in self.pkg.get('depends', []):
command = command + "-d %s " % dep
for provides in onlu.sflatten(self.pkg.get('provides', [])):
command = command + "--provides %s " % provides
if 'init' in self.pkg:
if not os.path.exists(self.pkg['init']):
raise OnlPackageError("Init script '%s' does not exist." %
self.pkg['init'])
command = command + "--deb-init %s " % self.pkg['init']
if 'post-install' in self.pkg:
if not os.path.exists(self.pkg['post-install']):
raise OnlPackageError(
"Post-install script '%s' does not exist." %
self.pkg['post-install'])
command = command + "--after-install %s " % self.pkg['post-install']
if logger.level < logging.INFO:
command = command + "--verbose "
onlu.execute(command)
# Grab the package from the workdir. There can be only one.
sys.stdout.write(workdir)
files = glob.glob(os.path.join(workdir, '*.deb'))
if len(files) == 0:
raise OnlPackageError("No debian package.")
elif len(files) > 1:
raise OnlPackageError("Too many packages.")
else:
# Move to the target directory
shutil.copy(files[0], dir_)
# Remove entire work directory.
shutil.rmtree(workdir)
# Return the path to the final built package
return os.path.join(dir_, os.path.basename(files[0]))
if ops.msconfig:
x.msconfig(ops.msconfig)
sys.exit(0)
if ops.show_packages:
print "\n".join(x.get_packages())
sys.exit(0)
if ops.dir is None:
raise OnlRfsError("argument --dir is required")
if not ops.no_build_packages:
pkgs = x.get_packages()
# Invoke onlpm to build all required (local) packages.
onlu.execute("onlpm --try-arches %s all --skip-missing --require %s" % (ops.arch, " ".join(pkgs)),
ex=OnlRfsError("Failed to build all required packages."))
if ops.multistrap_only:
x.multistrap(ops.dir)
sys.exit(0)
if not ops.no_multistrap and not os.getenv('NO_MULTISTRAP'):
x.multistrap(ops.dir)
x.configure(ops.dir)
if ops.cpio:
if onlu.execute("make-cpio.sh %s %s" % (ops.dir, ops.cpio)) != 0:
raise OnlRfsError("cpio creation failed.")
if ops.msconfig:
x.msconfig(ops.msconfig)
sys.exit(0)
if ops.show_packages:
print "\n".join(x.get_packages())
sys.exit(0)
if ops.dir is None:
raise OnlRfsError("argument --dir is required")
if not ops.no_build_packages:
pkgs = x.get_packages()
# Invoke onlpm to build all required (local) packages.
onlu.execute(
"%s/tools/onlpm.py --try-arches %s all --skip-missing --require %s"
% (os.getenv('ONL'), ops.arch, " ".join(pkgs)),
ex=OnlRfsError("Failed to build all required packages."))
if ops.only_build_packages:
sys.exit(0)
if ops.multistrap_only:
x.multistrap(ops.dir)
sys.exit(0)
if not ops.no_multistrap and not os.getenv('NO_MULTISTRAP'):
x.multistrap(ops.dir)
if not ops.no_configure and not os.getenv('NO_DPKG_CONFIGURE'):
x.configure(ops.dir)
if ops.update:
def user_shell_set(self, username, shell):
onlu.execute('chsh --shell %s %s' % (shell, username), chroot=self.chroot,
ex=OnlRfsError("Error setting shell %s for user %s" % (shell, username)))
def chown(file_, ownspec):
onlu.execute("sudo chown %s %s" % (ownspec, file_),
ex=OnlRfsError("Could not change ownership (%s) on file %s" % (ownspec, file_)))
def chmod(mode, file_):
onlu.execute("sudo chmod %s %s" % (mode, file_),
ex=OnlRfsError("Could not change permissions (%s) on file %s" % (mode, file_)))
def configure(self, dir_):
try:
onlu.execute(
"sudo mount -t devtmpfs dev %s" % os.path.join(dir_, "dev"),
ex=OnlRfsError("Could not mount dev in new filesystem."))
onlu.execute(
"sudo mount -t proc proc %s" % os.path.join(dir_, "proc"),
ex=OnlRfsError("Could not mount proc in new filesystem."))
script = os.path.join(dir_, "tmp/configure.sh")
if not os.getenv('NO_DPKG_CONFIGURE'):
self.dpkg_configure(dir_)
Configure = self.config.get('Configure', None)
if Configure:
for overlay in Configure.get('overlays', []):
logger.info("Overlay %s..." % overlay)
onlu.execute(
'tar -C %s -c --exclude "*~" . | sudo tar -C %s -x -v --no-same-owner'
% (overlay, dir_),
ex=OnlRfsError("Overlay '%s' failed." % overlay))
for update in Configure.get('update-rc.d', []):
onlu.execute("sudo chroot %s /usr/sbin/update-rc.d %s" %
(dir_, update),
ex=OnlRfsError("update-rc.d %s failed." %
(update)))
for script in Configure.get('scripts', []):
logger.info("Configuration script %s..." % script)
onlu.execute("sudo %s %s" % (script, dir_),
ex=OnlRfsError("script '%s' failed." %
script))
for command in Configure.get('commands', []):
if '__rfs__' in command:
command = command % dict(__rfs__=dir_)
logger.info("Configuration command '%s'..." % command)
onlu.execute(command,
ex=OnlRfsError("Command '%s' failed." %
command))
for (user, values) in Configure.get('users', {}).iteritems():
ua = OnlRfsSystemAdmin(dir_)
if user == 'root':
if 'password' in values:
ua.user_password_set(user, values['password'])
else:
ua.useradd(username=user, **values)
options = Configure.get('options', {})
if options.get('clean', False):
logger.info("Cleaning Filesystem...")
onlu.execute('sudo chroot %s /usr/bin/apt-get clean' %
dir_)
onlu.execute('sudo chroot %s /usr/sbin/localepurge' % dir_)
onlu.execute(
'sudo chroot %s find /usr/share/doc -type f -delete' %
dir_)
onlu.execute(
'sudo chroot %s find /usr/share/man -type f -delete' %
dir_)
if 'PermitRootLogin' in options:
config = os.path.join(dir_, 'etc/ssh/sshd_config')
ua.chmod('a+rw', config)
lines = open(config).readlines()
with open(config, "w") as f:
for line in lines:
if line.startswith('PermitRootLogin'):
v = options['PermitRootLogin']
logger.info("Setting PermitRootLogin to %s" %
v)
f.write('PermitRootLogin %s\n' % v)
else:
f.write(line)
ua.chmod('644', config)
if not options.get('securetty', True):
f = os.path.join(dir_, 'etc/securetty')
if os.path.exists(f):
logger.info("Removing %s" % f)
onlu.execute('sudo rm %s' % f,
ex=OnlRfsError(
'Could not remove file %s' % f))
if not options.get('ttys', False):
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
logger.info("Clearing %s ttys..." % f)
for line in fileinput.input(f, inplace=True):
if re.match("^[123456]:.*", line):
line = "#" + line
print line,
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('console', True):
logger.info('Configuring Console Access in %s' % f)
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
with open(f, 'a') as h:
h.write("T0:23:respawn:/sbin/pgetty\n")
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
for (mf, fields) in Configure.get('manifests', {}).iteritems():
logger.info("Configuring manifest %s..." % mf)
if mf.startswith('/'):
mf = mf[1:]
mname = os.path.join(dir_, mf)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(mname))
onlu.execute("sudo touch %s" % mname)
onlu.execute("sudo chmod a+w %s" % mname)
md = {}
md['version'] = json.load(open(fields['version']))
md['arch'] = self.arch
if os.path.exists(fields['platforms']):
md['platforms'] = yaml.load(open(fields['platforms']))
else:
md['platforms'] = fields['platforms'].split(',')
with open(mname, "w") as f:
json.dump(md, f, indent=2)
onlu.execute("sudo chmod a-w %s" % mname)
for (fname, v) in Configure.get('files',
{}).get('add', {}).iteritems():
if fname.startswith('/'):
fname = fname[1:]
dst = os.path.join(dir_, fname)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(dst))
onlu.execute("sudo touch %s" % dst)
onlu.execute("sudo chmod a+w %s" % dst)
if os.path.exists(v):
shutil.copy(v, dst)
else:
with open(dst, "w") as f:
f.write("%s\n" % v)
for fname in Configure.get('files', {}).get('remove', []):
if fname.startswith('/'):
fname = fname[1:]
f = os.path.join(dir_, fname)
if os.path.exists(f):
onlu.execute("sudo rm -rf %s" % f)
if Configure.get('issue'):
issue = Configure.get('issue')
fn = os.path.join(dir_, "etc/issue")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
fn = os.path.join(dir_, "etc/issue.net")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
finally:
onlu.execute(
"sudo umount -l %s %s" %
(os.path.join(dir_, "dev"), os.path.join(dir_, "proc")))
def build(self, dir_=None):
"""Build the debian package.
When this method is called it is assumed that all file
prerequisites for the package have already been built
or met. This is simply the packaging stage.
'dir_' : This is the output directory in which the package
should be left. If unspecified the package's local directory
will contain the package file."""
if "external" in self.pkg:
# Package file built externally
epkg = self.pkg["external"]
if os.path.exists(epkg):
return epkg
else:
raise OnlPackageError("The external package file '%s' does not exist." % epkg)
# Make sure all required files exist
if "files" in self.pkg:
self._validate_files()
# If dir_ is not specified, leave package in local package directory.
if dir_ is None:
dir_ = self.dir
workdir = tempfile.mkdtemp()
root = os.path.join(workdir, "root")
os.mkdir(root)
# The package file will be built into the workdir
self.pkg["__workdir"] = workdir
for (src, dst) in self.pkg.get("files", {}):
OnlPackage.copyf(src, dst, root)
for (link, src) in self.pkg.get("links", {}).iteritems():
logger.info("Linking %s -> %s..." % (link, src))
link = os.path.join(root, link)
os.symlink(src, link)
#
# FPM doesn't seem to have a doc option so we copy documentation
# files directly into place.
#
docpath = os.path.join(root, "usr/share/doc/%(name)s" % self.pkg)
if not os.path.exists(docpath):
os.makedirs(docpath)
for src in self.pkg.get("docs", []):
if not os.path.exists(src):
raise OnlPackageError("Documentation source file '%s' does not exist." % src)
shutil.copy(src, docpath)
changelog = os.path.join(workdir, "changelog")
copyright_ = os.path.join(workdir, "copyright")
#
# Export changelog and copyright files from the PKG dict
# to the workdir for processing.
#
# The copyright and changelog data can be embedded directly
# int the PKG file or kept as separate files.
#
def copy_str_or_file(src, dst):
if os.path.exists(src):
shutil.copyfile(src, dst)
else:
with open(dst, "w") as f:
f.write(src)
f.write("\n")
copy_str_or_file(self.pkg["copyright"], copyright_)
copy_str_or_file(self.pkg["changelog"], changelog)
############################################################
#
# Invoke fpm with all necessary options.
#
############################################################
self.pkg["__root"] = root
command = (
"""fpm -p %(__workdir)s -f -C %(__root)s -s dir -t deb -n %(name)s -v %(version)s -a %(arch)s -m %(maintainer)s --description "%(description)s" --url "%(url)s" --license "%(license)s" --vendor "%(vendor)s" """
% self.pkg
)
for dep in self.pkg.get("depends", []):
command = command + "-d %s " % dep
for provides in onlu.sflatten(self.pkg.get("provides", [])):
command = command + "--provides %s " % provides
for conflicts in onlu.sflatten(self.pkg.get("conflicts", [])):
command = command + "--conflicts %s " % conflicts
for replaces in onlu.sflatten(self.pkg.get("replaces", [])):
command = command + "--replaces %s " % replaces
if "virtual" in self.pkg:
command = command + "--provides %(v)s --conflicts %(v)s --replaces %(v)s " % dict(v=self.pkg["virtual"])
if "priority" in self.pkg:
command = command + "--deb-priority %s " % self.pkg["priority"]
if "init" in self.pkg:
if not os.path.exists(self.pkg["init"]):
raise OnlPackageError("Init script '%s' does not exist." % self.pkg["init"])
command = command + "--deb-init %s " % self.pkg["init"]
if self.pkg.get("init-after-install", True):
command = (
command + "--after-install %s " % OnlPackageAfterInstallScript(self.pkg["init"], dir=workdir).name
)
if self.pkg.get("init-before-remove", True):
command = (
command + "--before-remove %s " % OnlPackageBeforeRemoveScript(self.pkg["init"], dir=workdir).name
)
if self.pkg.get("init-after-remove", True):
command = (
command + "--after-remove %s " % OnlPackageAfterRemoveScript(self.pkg["init"], dir=workdir).name
)
if self.pkg.get("asr", True):
# Generate the ASR documentation for this package.
sys.path.append("%s/sm/infra/tools" % os.getenv("ONL"))
import asr
asro = asr.AimSyslogReference()
asro.extract(workdir)
asro.format(os.path.join(docpath, asr.AimSyslogReference.ASR_NAME), "json")
############################################################
if logger.level < logging.INFO:
command = command + "--verbose "
onlu.execute(command)
# Grab the package from the workdir. There can be only one.
files = glob.glob(os.path.join(workdir, "*.deb"))
if len(files) == 0:
raise OnlPackageError("No debian package.")
elif len(files) > 1:
raise OnlPackageError("Too many packages.")
else:
# Move to the target directory
shutil.copy(files[0], dir_)
# Remove entire work directory.
shutil.rmtree(workdir)
# Return the path to the final built package
return os.path.join(dir_, os.path.basename(files[0]))
def configure(self, dir_):
try:
onlu.execute("sudo mount -t devtmpfs dev %s" % os.path.join(dir_, "dev"),
ex=OnlRfsError("Could not mount dev in new filesystem."))
onlu.execute("sudo mount -t proc proc %s" % os.path.join(dir_, "proc"),
ex=OnlRfsError("Could not mount proc in new filesystem."))
script = os.path.join(dir_, "tmp/configure.sh")
if not os.getenv('NO_DPKG_CONFIGURE'):
self.dpkg_configure(dir_)
os_release = os.path.join(dir_, 'etc', 'os-release')
if os.path.exists(os_release):
# Convert /etc/os-release to /etc/os-release.json
import shlex
contents = open(os_release).read()
d = dict(token.split('=') for token in shlex.split(contents))
ua = OnlRfsSystemAdmin(dir_)
ua.chmod('a+rwx', os.path.dirname(os_release))
with open(os.path.join(os.path.dirname(os_release), 'os-release.json'), "w") as f:
f.write(json.dumps(d))
ua.chmod('0755', os.path.dirname(os_release))
Configure = self.config.get('Configure', None)
if Configure:
for overlay in Configure.get('overlays', []):
logger.info("Overlay %s..." % overlay)
onlu.execute('tar -C %s -c --exclude "*~" . | sudo tar -C %s -x -v --no-same-owner' % (overlay, dir_),
ex=OnlRfsError("Overlay '%s' failed." % overlay))
for update in Configure.get('update-rc.d', []):
onlu.execute("sudo chroot %s /usr/sbin/update-rc.d %s" % (dir_, update),
ex=OnlRfsError("update-rc.d %s failed." % (update)))
for script in Configure.get('scripts', []):
logger.info("Configuration script %s..." % script)
onlu.execute("sudo %s %s" % (script, dir_),
ex=OnlRfsError("script '%s' failed." % script))
for command in Configure.get('commands', []):
if '__rfs__' in command:
command = command % dict(__rfs__=dir_)
logger.info("Configuration command '%s'..." % command)
onlu.execute(command,
ex=OnlRfsError("Command '%s' failed." % command))
for (user, values) in Configure.get('users', {}).iteritems():
ua = OnlRfsSystemAdmin(dir_)
if user == 'root':
if 'password' in values:
ua.user_password_set(user, values['password'])
else:
ua.useradd(username=user, **values)
options = Configure.get('options', {})
if options.get('clean', False):
logger.info("Cleaning Filesystem...")
onlu.execute('sudo chroot %s /usr/bin/apt-get clean' % dir_)
onlu.execute('sudo chroot %s /usr/sbin/localepurge' % dir_ )
onlu.execute('sudo chroot %s find /usr/share/doc -type f -not -name asr.json -delete' % dir_)
onlu.execute('sudo chroot %s find /usr/share/man -type f -delete' % dir_)
if 'PermitRootLogin' in options:
config = os.path.join(dir_, 'etc/ssh/sshd_config')
ua.chmod('a+rw', config)
lines = open(config).readlines()
with open(config, "w") as f:
for line in lines:
if line.startswith('PermitRootLogin'):
v = options['PermitRootLogin']
logger.info("Setting PermitRootLogin to %s" % v)
f.write('PermitRootLogin %s\n' % v)
else:
f.write(line)
ua.chmod('644', config)
if not options.get('securetty', True):
f = os.path.join(dir_, 'etc/securetty')
if os.path.exists(f):
logger.info("Removing %s" % f)
onlu.execute('sudo rm %s' % f,
ex=OnlRfsError('Could not remove file %s' % f))
if not options.get('ttys', False):
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
logger.info("Clearing %s ttys..." % f)
for line in fileinput.input(f, inplace=True):
if re.match("^[123456]:.*", line):
line = "#" + line
print line,
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('console', True):
logger.info('Configuring Console Access in %s' % f)
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
with open(f, 'a') as h:
h.write("T0:23:respawn:/sbin/pgetty\n")
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('asr', None):
asropts = options.get('asr')
logger.info("Gathering ASR documentation...")
sys.path.append("%s/sm/infra/tools" % os.getenv('ONL'))
import asr
asro = asr.AimSyslogReference()
asro.merge(dir_)
asro.format(os.path.join(dir_, asropts['file']), fmt=asropts['format'])
for (mf, fields) in Configure.get('manifests', {}).iteritems():
logger.info("Configuring manifest %s..." % mf)
if mf.startswith('/'):
mf = mf[1:]
mname = os.path.join(dir_, mf)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(mname))
onlu.execute("sudo touch %s" % mname)
onlu.execute("sudo chmod a+w %s" % mname)
md = {}
md['version'] = json.load(open(fields['version']))
md['arch'] = self.arch
if os.path.exists(fields['platforms']):
md['platforms'] = yaml.load(open(fields['platforms']))
else:
md['platforms'] = fields['platforms'].split(',')
for (k, v) in fields.get('keys', {}).iteritems():
if k in md:
md[k].update(v)
else:
md[k] = v
with open(mname, "w") as f:
json.dump(md, f, indent=2)
onlu.execute("sudo chmod a-w %s" % mname)
for (fname, v) in Configure.get('files', {}).get('add', {}).iteritems():
if fname.startswith('/'):
fname = fname[1:]
dst = os.path.join(dir_, fname)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(dst))
onlu.execute("sudo touch %s" % dst)
onlu.execute("sudo chmod a+w %s" % dst)
if os.path.exists(v):
shutil.copy(v, dst)
else:
with open(dst, "w") as f:
f.write("%s\n" % v)
for fname in Configure.get('files', {}).get('remove', []):
if fname.startswith('/'):
fname = fname[1:]
f = os.path.join(dir_, fname)
if os.path.exists(f):
onlu.execute("sudo rm -rf %s" % f)
if Configure.get('issue'):
issue = Configure.get('issue')
fn = os.path.join(dir_, "etc/issue")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
fn = os.path.join(dir_, "etc/issue.net")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
finally:
onlu.execute("sudo umount -l %s %s" % (os.path.join(dir_, "dev"), os.path.join(dir_, "proc")))
def contents(self, pkg):
with self.lock:
path = self.lookup(pkg)
if path:
print "** %s contents:" % path
onlu.execute(["dpkg", "-c", path])
def configure(self, dir_):
if not os.getenv('NO_DPKG_CONFIGURE'):
with OnlRfsContext(dir_, resolvconf=False):
self.dpkg_configure(dir_)
with OnlRfsContext(dir_):
os_release = os.path.join(dir_, 'etc', 'os-release')
os_release_dict = {}
if os.path.exists(os_release):
# Convert /etc/os-release to /etc/os-release.json
import shlex
contents = open(os_release).read()
os_release_dict = dict(token.split('=') for token in shlex.split(contents))
ua = OnlRfsSystemAdmin(dir_)
ua.chmod('a+rwx', os.path.dirname(os_release))
with open(os.path.join(os.path.dirname(os_release), 'os-release.json'), "w") as f:
f.write(json.dumps(os_release_dict))
ua.chmod('0755', os.path.dirname(os_release))
Configure = self.config.get('Configure', None)
if Configure:
for cmd in Configure.get('run', []):
onlu.execute("sudo chroot %s %s" % (dir_, cmd),
ex=OnlRfsError("run command '%s' failed" % cmd))
for overlay in Configure.get('overlays', []):
logger.info("Overlay %s..." % overlay)
onlu.execute('tar -C %s -c --exclude "*~" . | sudo tar -C %s -x -v --no-same-owner' % (overlay, dir_),
ex=OnlRfsError("Overlay '%s' failed." % overlay))
for update in Configure.get('update-rc.d', []):
onlu.execute("sudo chroot %s /usr/sbin/update-rc.d %s" % (dir_, update),
ex=OnlRfsError("update-rc.d %s failed." % (update)))
for module in Configure.get('modules', []):
monfig = os.path.join(dir_, 'etc/modules')
ua.chmod('a+rw', monfig)
# This is a bad way to add the modules but works for now
onlu.execute("sudo chroot %s echo %s >> %s" % (dir_, module, monfig))
for sysctl in Configure.get('sysctl', []):
onlu.execute("sudo chroot %s /bin/systemctl %s" % (dir_, sysctl))
for script in Configure.get('scripts', []):
logger.info("Configuration script %s..." % script)
onlu.execute("sudo %s %s" % (script, dir_),
ex=OnlRfsError("script '%s' failed." % script))
for command in Configure.get('commands', []):
if '__rfs__' in command:
command = command % dict(__rfs__=dir_)
logger.info("Configuration command '%s'..." % command)
onlu.execute(command,
ex=OnlRfsError("Command '%s' failed." % command))
ua = OnlRfsSystemAdmin(dir_)
for (group, values) in Configure.get('groups', {}).iteritems():
ua.groupadd(group=group, **values if values else {})
for (user, values) in Configure.get('users', {}).iteritems():
if user == 'root':
if 'password' in values:
ua.user_password_set(user, values['password'])
else:
ua.useradd(username=user, **values)
options = Configure.get('options', {})
if options.get('clean', False):
logger.info("Cleaning Filesystem...")
onlu.execute('sudo chroot %s /usr/bin/apt-get clean' % dir_)
onlu.execute('sudo chroot %s /usr/sbin/localepurge' % dir_ )
onlu.execute('sudo chroot %s find /usr/share/doc -type f -not -name asr.json -delete' % dir_)
onlu.execute('sudo chroot %s find /usr/share/man -type f -delete' % dir_)
if 'PermitRootLogin' in options:
config = os.path.join(dir_, 'etc/ssh/sshd_config')
ua.chmod('a+rw', config)
lines = open(config).readlines()
with open(config, "w") as f:
for line in lines:
# Added the hash sign to update this function
if line.startswith('#PermitRootLogin'):
v = options['PermitRootLogin']
logger.info("Setting PermitRootLogin to %s" % v)
f.write('PermitRootLogin %s\n' % v)
else:
f.write(line)
ua.chmod('644', config)
if not options.get('securetty', True):
f = os.path.join(dir_, 'etc/securetty')
if os.path.exists(f):
logger.info("Removing %s" % f)
onlu.execute('sudo rm %s' % f,
ex=OnlRfsError('Could not remove file %s' % f))
if os.path.exists(os.path.join(dir_, 'etc/inittab')):
if not options.get('ttys', False):
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
logger.info("Clearing %s ttys..." % f)
for line in fileinput.input(f, inplace=True):
if re.match("^[123456]:.*", line):
line = "#" + line
print line,
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('console', True):
logger.info('Configuring Console Access in %s' % f)
f = os.path.join(dir_, 'etc/inittab')
ua.chmod('a+w', f)
ua.chmod('a+w', os.path.dirname(f))
with open(f, 'a') as h:
h.write("T0:23:respawn:/sbin/pgetty\n")
ua.chmod('go-w', f)
ua.chmod('go-w', os.path.dirname(f))
if options.get('asr', None):
asropts = options.get('asr')
logger.info("Gathering ASR documentation...")
sys.path.append("%s/sm/infra/tools" % os.getenv('ONL'))
import asr
asro = asr.AimSyslogReference()
asro.merge(dir_)
asrf = os.path.join(dir_, asropts['file'])
OnlRfsSystemAdmin.chmod('777', os.path.dirname(asrf))
asro.format(os.path.join(dir_, asropts['file']), fmt=asropts['format'])
for (mf, fields) in Configure.get('manifests', {}).iteritems():
logger.info("Configuring manifest %s..." % mf)
if mf.startswith('/'):
mf = mf[1:]
mname = os.path.join(dir_, mf)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(mname))
onlu.execute("sudo touch %s" % mname)
onlu.execute("sudo chmod a+w %s" % mname)
md = {}
md['version'] = json.load(open(fields['version']))
md['arch'] = self.arch
md['os-release'] = os_release_dict
if os.path.exists(fields['platforms']):
md['platforms'] = yaml.load(open(fields['platforms']))
else:
md['platforms'] = fields['platforms'].split(',')
for (k, v) in fields.get('keys', {}).iteritems():
if k in md:
md[k].update(v)
else:
md[k] = v
with open(mname, "w") as f:
json.dump(md, f, indent=2)
onlu.execute("sudo chmod a-w %s" % mname)
for (fname, v) in Configure.get('files', {}).get('add', {}).iteritems():
if fname.startswith('/'):
fname = fname[1:]
dst = os.path.join(dir_, fname)
onlu.execute("sudo mkdir -p %s" % os.path.dirname(dst))
onlu.execute("sudo touch %s" % dst)
onlu.execute("sudo chmod a+w %s" % dst)
if os.path.exists(v):
shutil.copy(v, dst)
else:
with open(dst, "w") as f:
f.write("%s\n" % v)
for fname in Configure.get('files', {}).get('remove', []):
if fname.startswith('/'):
fname = fname[1:]
f = os.path.join(dir_, fname)
if os.path.exists(f):
onlu.execute("sudo rm -rf %s" % f)
if Configure.get('issue'):
issue = Configure.get('issue')
fn = os.path.join(dir_, "etc/issue")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
fn = os.path.join(dir_, "etc/issue.net")
onlu.execute("sudo chmod a+w %s" % fn)
with open(fn, "w") as f:
f.write("%s\n" % issue)
onlu.execute("sudo chmod a-w %s" % fn)
if Configure.get('easy_install'):
# install Python2 eggs (with this version of Python)
# tee hee, we cannot use distutils.sysconfig.get_python_lib(),
# because ONL/Dent doesn't use the path layout as the host
pyver = "%d.%d" % (sys.version_info[0], sys.version_info[1],)
pyver = subprocess.check_output(cmd,
universal_newlines=True).strip()
pydir = "%s/usr/lib/python%s/dist-packages" % (dir_, pyver,)
bindir = "%s/usr/bin" % (dir_,)
cmd = "mkdir -p %s" % pydir
onlu.execute(cmd, sudo=True, ex=True)
for egg in Configure.get('easy_install3', []):
logger.info("Installing Egg %s", egg)
cmd = ('easy_install3',
'--always-unzip', '--no-deps',
'--install-dir', pydir,
'--script-dir', bindir,
egg,)
cmd = ("env PYTHONPATH=%s %s"
% (pydir, " ".join(cmd),))
onlu.execute(cmd, sudo=True, ex=True)
if Configure.get('easy_install3'):
# install Python3 eggs (foreign version of Python)
# tee hee, we cannot use distutils.sysconfig.get_python_lib(),
# because ONL/Dent doesn't use the path layout as the host
cmd = "import distutils.sysconfig; print(distutils.sysconfig.get_python_version())"
cmd = ('python3', '-c', cmd,)
pyver = subprocess.check_output(cmd,
universal_newlines=True).strip()
pydir = "%s/usr/lib/python%s/dist-packages" % (dir_, pyver,)
bindir = "%s/usr/bin" % (dir_,)
cmd = "mkdir -p %s" % pydir
onlu.execute(cmd, sudo=True, ex=True)
for egg in Configure.get('easy_install3', []):
logger.info("Installing Egg %s", egg)
cmd = ('easy_install3',
'--always-unzip', '--no-deps',
'--install-dir', pydir,
'--script-dir', bindir,
egg,)
cmd = ("env PYTHONPATH=%s %s"
% (pydir, " ".join(cmd),))
onlu.execute(cmd, sudo=True, ex=True)
