def build(self, dir_=None): """Build the debian package. When this method is called it is assumed that all file prerequisites for the package have already been built or met. This is simply the packaging stage. 'dir_' : This is the output directory in which the package should be left. If unspecified the package's local directory will contain the package file.""" if 'external' in self.pkg: # Package file built externally epkg = self.pkg['external'] if os.path.exists(epkg): return epkg else: raise OnlPackageError("The external package file '%s' does not exist." % epkg); # Make sure all required files exist if 'files' in self.pkg: self._validate_files('files', True) if 'optional-files' in self.pkg: self._validate_files('optional-files', False) # If dir_ is not specified, leave package in local package directory. if dir_ is None: dir_ = self.dir workdir = tempfile.mkdtemp() root = os.path.join(workdir, "root"); os.mkdir(root); # The package file will be built into the workdir self.pkg['__workdir'] = workdir for (src,dst) in self.pkg.get('files', {}): OnlPackage.copyf(src, dst, root) for (src,dst) in self.pkg.get('optional-files', {}): if os.path.exists(src): OnlPackage.copyf(src, dst, root) for (link,src) in self.pkg.get('links', {}).iteritems(): logger.info("Linking %s -> %s..." % (link, src)) link = os.path.join(root, link) os.symlink(src, link) # # FPM doesn't seem to have a doc option so we copy documentation # files directly into place. # docpath = os.path.join(root, "usr/share/doc/%(name)s" % self.pkg) if not os.path.exists(docpath): os.makedirs(docpath) for src in self.pkg.get('docs', []): if not os.path.exists(src): raise OnlPackageError("Documentation source file '%s' does not exist." % src) shutil.copy(src, docpath) changelog = os.path.join(workdir, 'changelog') copyright_ = os.path.join(workdir, 'copyright') # # Export changelog and copyright files from the PKG dict # to the workdir for processing. # # The copyright and changelog data can be embedded directly # int the PKG file or kept as separate files. # def copy_str_or_file(src, dst): if os.path.exists(src): shutil.copyfile(src, dst) else: with open(dst, "w") as f: f.write(src) f.write("\n") copy_str_or_file(self.pkg['copyright'], copyright_) copy_str_or_file(self.pkg['changelog'], changelog) ############################################################ # # Invoke fpm with all necessary options. # ############################################################ self.pkg['__root'] = root command = """fpm -p %(__workdir)s -f -C %(__root)s -s dir -t deb -n %(name)s -v %(version)s -a %(arch)s -m %(maintainer)s --description "%(description)s" --url "%(url)s" --license "%(license)s" --vendor "%(vendor)s" """ % self.pkg for dep in self.pkg.get('depends', []): command = command + "-d %s " % dep for provides in onlu.sflatten(self.pkg.get('provides', [])): command = command + "--provides %s " % provides for conflicts in onlu.sflatten(self.pkg.get('conflicts', [])): command = command + "--conflicts %s " % conflicts for replaces in onlu.sflatten(self.pkg.get('replaces', [])): command = command + "--replaces %s " % replaces if 'virtual' in self.pkg: command = command + "--provides %(v)s --conflicts %(v)s --replaces %(v)s " % dict(v=self.pkg['virtual']) if 'priority' in self.pkg: command = command + "--deb-priority %s " % self.pkg['priority'] if 'init' in self.pkg: if not os.path.exists(self.pkg['init']): raise OnlPackageError("Init script '%s' does not exist." % self.pkg['init']) command = command + "--deb-init %s " % self.pkg['init'] if self.pkg.get('init-after-install', True): command = command + "--after-install %s " % OnlPackageAfterInstallScript(self.pkg['init'], dir=workdir).name if self.pkg.get('init-before-remove', True): command = command + "--before-remove %s " % OnlPackageBeforeRemoveScript(self.pkg['init'], dir=workdir).name if self.pkg.get('init-after-remove', True): command = command + "--after-remove %s " % OnlPackageAfterRemoveScript(self.pkg['init'], dir=workdir).name if self.pkg.get('asr', False): with onlu.Profiler() as profiler: # Generate the ASR documentation for this package. sys.path.append("%s/sm/infra/tools" % os.getenv('ONL')) import asr asro = asr.AimSyslogReference() asro.extract(workdir) asro.format(os.path.join(docpath, asr.AimSyslogReference.ASR_NAME), 'json') profiler.log("ASR generation for %(name)s" % self.pkg) ############################################################ if logger.level < logging.INFO: command = command + "--verbose " onlu.execute(command) # Grab the package from the workdir. There can be only one. files = glob.glob(os.path.join(workdir, '*.deb')) if len(files) == 0: raise OnlPackageError("No debian package.") elif len(files) > 1: raise OnlPackageError("Too many packages.") else: # Move to the target directory shutil.copy(files[0], dir_) # Remove entire work directory. shutil.rmtree(workdir) # Return the path to the final built package return os.path.join(dir_, os.path.basename(files[0]))
def configure(self, dir_): if not os.getenv('NO_DPKG_CONFIGURE'): with OnlRfsContext(dir_, resolvconf=False): self.dpkg_configure(dir_) with OnlRfsContext(dir_): os_release = os.path.join(dir_, 'etc', 'os-release') if os.path.exists(os_release): # Convert /etc/os-release to /etc/os-release.json import shlex contents = open(os_release).read() d = dict(token.split('=') for token in shlex.split(contents)) ua = OnlRfsSystemAdmin(dir_) ua.chmod('a+rwx', os.path.dirname(os_release)) with open( os.path.join(os.path.dirname(os_release), 'os-release.json'), "w") as f: f.write(json.dumps(d)) ua.chmod('0755', os.path.dirname(os_release)) Configure = self.config.get('Configure', None) if Configure: for cmd in Configure.get('run', []): onlu.execute("sudo chroot %s %s" % (dir_, cmd), ex=OnlRfsError("run command '%s' failed" % cmd)) for overlay in Configure.get('overlays', []): logger.info("Overlay %s..." % overlay) onlu.execute( 'tar -C %s -c --exclude "*~" . | sudo tar -C %s -x -v --no-same-owner' % (overlay, dir_), ex=OnlRfsError("Overlay '%s' failed." % overlay)) for update in Configure.get('update-rc.d', []): onlu.execute("sudo chroot %s /usr/sbin/update-rc.d %s" % (dir_, update), ex=OnlRfsError("update-rc.d %s failed." % (update))) for script in Configure.get('scripts', []): logger.info("Configuration script %s..." % script) onlu.execute("sudo %s %s" % (script, dir_), ex=OnlRfsError("script '%s' failed." % script)) for command in Configure.get('commands', []): if '__rfs__' in command: command = command % dict(__rfs__=dir_) logger.info("Configuration command '%s'..." % command) onlu.execute(command, ex=OnlRfsError("Command '%s' failed." % command)) ua = OnlRfsSystemAdmin(dir_) for (group, values) in Configure.get('groups', {}).iteritems(): ua.groupadd(group=group, **values if values else {}) for (user, values) in Configure.get('users', {}).iteritems(): if user == 'root': if 'password' in values: ua.user_password_set(user, values['password']) else: ua.useradd(username=user, **values) options = Configure.get('options', {}) if options.get('clean', False): logger.info("Cleaning Filesystem...") onlu.execute('sudo chroot %s /usr/bin/apt-get clean' % dir_) onlu.execute('sudo chroot %s /usr/sbin/localepurge' % dir_) onlu.execute( 'sudo chroot %s find /usr/share/doc -type f -not -name asr.json -delete' % dir_) onlu.execute( 'sudo chroot %s find /usr/share/man -type f -delete' % dir_) if 'PermitRootLogin' in options: config = os.path.join(dir_, 'etc/ssh/sshd_config') ua.chmod('a+rw', config) lines = open(config).readlines() with open(config, "w") as f: for line in lines: if line.startswith('PermitRootLogin'): v = options['PermitRootLogin'] logger.info("Setting PermitRootLogin to %s" % v) f.write('PermitRootLogin %s\n' % v) else: f.write(line) ua.chmod('644', config) if not options.get('securetty', True): f = os.path.join(dir_, 'etc/securetty') if os.path.exists(f): logger.info("Removing %s" % f) onlu.execute('sudo rm %s' % f, ex=OnlRfsError( 'Could not remove file %s' % f)) if not options.get('ttys', False): f = os.path.join(dir_, 'etc/inittab') ua.chmod('a+w', f) ua.chmod('a+w', os.path.dirname(f)) logger.info("Clearing %s ttys..." % f) for line in fileinput.input(f, inplace=True): if re.match("^[123456]:.*", line): line = "#" + line print line, ua.chmod('go-w', f) ua.chmod('go-w', os.path.dirname(f)) if options.get('console', True): logger.info('Configuring Console Access in %s' % f) f = os.path.join(dir_, 'etc/inittab') ua.chmod('a+w', f) ua.chmod('a+w', os.path.dirname(f)) with open(f, 'a') as h: h.write("T0:23:respawn:/sbin/pgetty\n") ua.chmod('go-w', f) ua.chmod('go-w', os.path.dirname(f)) if options.get('asr', None): asropts = options.get('asr') logger.info("Gathering ASR documentation...") sys.path.append("%s/sm/infra/tools" % os.getenv('ONL')) import asr asro = asr.AimSyslogReference() asro.merge(dir_) asro.format(os.path.join(dir_, asropts['file']), fmt=asropts['format']) for (mf, fields) in Configure.get('manifests', {}).iteritems(): logger.info("Configuring manifest %s..." % mf) if mf.startswith('/'): mf = mf[1:] mname = os.path.join(dir_, mf) onlu.execute("sudo mkdir -p %s" % os.path.dirname(mname)) onlu.execute("sudo touch %s" % mname) onlu.execute("sudo chmod a+w %s" % mname) md = {} md['version'] = json.load(open(fields['version'])) md['arch'] = self.arch if os.path.exists(fields['platforms']): md['platforms'] = yaml.load(open(fields['platforms'])) else: md['platforms'] = fields['platforms'].split(',') for (k, v) in fields.get('keys', {}).iteritems(): if k in md: md[k].update(v) else: md[k] = v with open(mname, "w") as f: json.dump(md, f, indent=2) onlu.execute("sudo chmod a-w %s" % mname) for (fname, v) in Configure.get('files', {}).get('add', {}).iteritems(): if fname.startswith('/'): fname = fname[1:] dst = os.path.join(dir_, fname) onlu.execute("sudo mkdir -p %s" % os.path.dirname(dst)) onlu.execute("sudo touch %s" % dst) onlu.execute("sudo chmod a+w %s" % dst) if os.path.exists(v): shutil.copy(v, dst) else: with open(dst, "w") as f: f.write("%s\n" % v) for fname in Configure.get('files', {}).get('remove', []): if fname.startswith('/'): fname = fname[1:] f = os.path.join(dir_, fname) if os.path.exists(f): onlu.execute("sudo rm -rf %s" % f) if Configure.get('issue'): issue = Configure.get('issue') fn = os.path.join(dir_, "etc/issue") onlu.execute("sudo chmod a+w %s" % fn) with open(fn, "w") as f: f.write("%s\n\n" % issue) onlu.execute("sudo chmod a-w %s" % fn) fn = os.path.join(dir_, "etc/issue.net") onlu.execute("sudo chmod a+w %s" % fn) with open(fn, "w") as f: f.write("%s\n" % issue) onlu.execute("sudo chmod a-w %s" % fn)
def configure(self, dir_): if not os.getenv('NO_DPKG_CONFIGURE'): with OnlRfsContext(dir_, resolvconf=False): self.dpkg_configure(dir_) with OnlRfsContext(dir_): os_release = os.path.join(dir_, 'etc', 'os-release') os_release_dict = {} if os.path.exists(os_release): # Convert /etc/os-release to /etc/os-release.json import shlex contents = open(os_release).read() os_release_dict = dict(token.split('=') for token in shlex.split(contents)) ua = OnlRfsSystemAdmin(dir_) ua.chmod('a+rwx', os.path.dirname(os_release)) with open(os.path.join(os.path.dirname(os_release), 'os-release.json'), "w") as f: f.write(json.dumps(os_release_dict)) ua.chmod('0755', os.path.dirname(os_release)) Configure = self.config.get('Configure', None) if Configure: for cmd in Configure.get('run', []): onlu.execute("sudo chroot %s %s" % (dir_, cmd), ex=OnlRfsError("run command '%s' failed" % cmd)) for overlay in Configure.get('overlays', []): logger.info("Overlay %s..." % overlay) onlu.execute('tar -C %s -c --exclude "*~" . | sudo tar -C %s -x -v --no-same-owner' % (overlay, dir_), ex=OnlRfsError("Overlay '%s' failed." % overlay)) for update in Configure.get('update-rc.d', []): onlu.execute("sudo chroot %s /usr/sbin/update-rc.d %s" % (dir_, update), ex=OnlRfsError("update-rc.d %s failed." % (update))) for module in Configure.get('modules', []): monfig = os.path.join(dir_, 'etc/modules') ua.chmod('a+rw', monfig) # This is a bad way to add the modules but works for now onlu.execute("sudo chroot %s echo %s >> %s" % (dir_, module, monfig)) for sysctl in Configure.get('sysctl', []): onlu.execute("sudo chroot %s /bin/systemctl %s" % (dir_, sysctl)) for script in Configure.get('scripts', []): logger.info("Configuration script %s..." % script) onlu.execute("sudo %s %s" % (script, dir_), ex=OnlRfsError("script '%s' failed." % script)) for command in Configure.get('commands', []): if '__rfs__' in command: command = command % dict(__rfs__=dir_) logger.info("Configuration command '%s'..." % command) onlu.execute(command, ex=OnlRfsError("Command '%s' failed." % command)) ua = OnlRfsSystemAdmin(dir_) for (group, values) in Configure.get('groups', {}).iteritems(): ua.groupadd(group=group, **values if values else {}) for (user, values) in Configure.get('users', {}).iteritems(): if user == 'root': if 'password' in values: ua.user_password_set(user, values['password']) else: ua.useradd(username=user, **values) options = Configure.get('options', {}) if options.get('clean', False): logger.info("Cleaning Filesystem...") onlu.execute('sudo chroot %s /usr/bin/apt-get clean' % dir_) onlu.execute('sudo chroot %s /usr/sbin/localepurge' % dir_ ) onlu.execute('sudo chroot %s find /usr/share/doc -type f -not -name asr.json -delete' % dir_) onlu.execute('sudo chroot %s find /usr/share/man -type f -delete' % dir_) if 'PermitRootLogin' in options: config = os.path.join(dir_, 'etc/ssh/sshd_config') ua.chmod('a+rw', config) lines = open(config).readlines() with open(config, "w") as f: for line in lines: # Added the hash sign to update this function if line.startswith('#PermitRootLogin'): v = options['PermitRootLogin'] logger.info("Setting PermitRootLogin to %s" % v) f.write('PermitRootLogin %s\n' % v) else: f.write(line) ua.chmod('644', config) if not options.get('securetty', True): f = os.path.join(dir_, 'etc/securetty') if os.path.exists(f): logger.info("Removing %s" % f) onlu.execute('sudo rm %s' % f, ex=OnlRfsError('Could not remove file %s' % f)) if os.path.exists(os.path.join(dir_, 'etc/inittab')): if not options.get('ttys', False): f = os.path.join(dir_, 'etc/inittab') ua.chmod('a+w', f) ua.chmod('a+w', os.path.dirname(f)) logger.info("Clearing %s ttys..." % f) for line in fileinput.input(f, inplace=True): if re.match("^[123456]:.*", line): line = "#" + line print line, ua.chmod('go-w', f) ua.chmod('go-w', os.path.dirname(f)) if options.get('console', True): logger.info('Configuring Console Access in %s' % f) f = os.path.join(dir_, 'etc/inittab') ua.chmod('a+w', f) ua.chmod('a+w', os.path.dirname(f)) with open(f, 'a') as h: h.write("T0:23:respawn:/sbin/pgetty\n") ua.chmod('go-w', f) ua.chmod('go-w', os.path.dirname(f)) if options.get('asr', None): asropts = options.get('asr') logger.info("Gathering ASR documentation...") sys.path.append("%s/sm/infra/tools" % os.getenv('ONL')) import asr asro = asr.AimSyslogReference() asro.merge(dir_) asrf = os.path.join(dir_, asropts['file']) OnlRfsSystemAdmin.chmod('777', os.path.dirname(asrf)) asro.format(os.path.join(dir_, asropts['file']), fmt=asropts['format']) for (mf, fields) in Configure.get('manifests', {}).iteritems(): logger.info("Configuring manifest %s..." % mf) if mf.startswith('/'): mf = mf[1:] mname = os.path.join(dir_, mf) onlu.execute("sudo mkdir -p %s" % os.path.dirname(mname)) onlu.execute("sudo touch %s" % mname) onlu.execute("sudo chmod a+w %s" % mname) md = {} md['version'] = json.load(open(fields['version'])) md['arch'] = self.arch md['os-release'] = os_release_dict if os.path.exists(fields['platforms']): md['platforms'] = yaml.load(open(fields['platforms'])) else: md['platforms'] = fields['platforms'].split(',') for (k, v) in fields.get('keys', {}).iteritems(): if k in md: md[k].update(v) else: md[k] = v with open(mname, "w") as f: json.dump(md, f, indent=2) onlu.execute("sudo chmod a-w %s" % mname) for (fname, v) in Configure.get('files', {}).get('add', {}).iteritems(): if fname.startswith('/'): fname = fname[1:] dst = os.path.join(dir_, fname) onlu.execute("sudo mkdir -p %s" % os.path.dirname(dst)) onlu.execute("sudo touch %s" % dst) onlu.execute("sudo chmod a+w %s" % dst) if os.path.exists(v): shutil.copy(v, dst) else: with open(dst, "w") as f: f.write("%s\n" % v) for fname in Configure.get('files', {}).get('remove', []): if fname.startswith('/'): fname = fname[1:] f = os.path.join(dir_, fname) if os.path.exists(f): onlu.execute("sudo rm -rf %s" % f) if Configure.get('issue'): issue = Configure.get('issue') fn = os.path.join(dir_, "etc/issue") onlu.execute("sudo chmod a+w %s" % fn) with open(fn, "w") as f: f.write("%s\n\n" % issue) onlu.execute("sudo chmod a-w %s" % fn) fn = os.path.join(dir_, "etc/issue.net") onlu.execute("sudo chmod a+w %s" % fn) with open(fn, "w") as f: f.write("%s\n" % issue) onlu.execute("sudo chmod a-w %s" % fn) if Configure.get('easy_install'): # install Python2 eggs (with this version of Python) # tee hee, we cannot use distutils.sysconfig.get_python_lib(), # because ONL/Dent doesn't use the path layout as the host pyver = "%d.%d" % (sys.version_info[0], sys.version_info[1],) pyver = subprocess.check_output(cmd, universal_newlines=True).strip() pydir = "%s/usr/lib/python%s/dist-packages" % (dir_, pyver,) bindir = "%s/usr/bin" % (dir_,) cmd = "mkdir -p %s" % pydir onlu.execute(cmd, sudo=True, ex=True) for egg in Configure.get('easy_install3', []): logger.info("Installing Egg %s", egg) cmd = ('easy_install3', '--always-unzip', '--no-deps', '--install-dir', pydir, '--script-dir', bindir, egg,) cmd = ("env PYTHONPATH=%s %s" % (pydir, " ".join(cmd),)) onlu.execute(cmd, sudo=True, ex=True) if Configure.get('easy_install3'): # install Python3 eggs (foreign version of Python) # tee hee, we cannot use distutils.sysconfig.get_python_lib(), # because ONL/Dent doesn't use the path layout as the host cmd = "import distutils.sysconfig; print(distutils.sysconfig.get_python_version())" cmd = ('python3', '-c', cmd,) pyver = subprocess.check_output(cmd, universal_newlines=True).strip() pydir = "%s/usr/lib/python%s/dist-packages" % (dir_, pyver,) bindir = "%s/usr/bin" % (dir_,) cmd = "mkdir -p %s" % pydir onlu.execute(cmd, sudo=True, ex=True) for egg in Configure.get('easy_install3', []): logger.info("Installing Egg %s", egg) cmd = ('easy_install3', '--always-unzip', '--no-deps', '--install-dir', pydir, '--script-dir', bindir, egg,) cmd = ("env PYTHONPATH=%s %s" % (pydir, " ".join(cmd),)) onlu.execute(cmd, sudo=True, ex=True)