def set_project_version(self, builddir, new_version=None): if new_version == "": raise ElbeDBError("version number must not be empty") if not re.match("^[A-Za-z0-9_.-]{1,25}$", new_version): raise ElbeDBError( "version number must contain valid characters [A-Za-z0-9_-.]") with session_scope(self.session) as s: try: p = s.query( Project ).filter( Project.builddir == builddir ).\ one() except NoResultFound: raise ElbeDBError( "project %s is not registered in the database" % builddir) if p.status == "empty_project" or p.status == "busy": raise ElbeDBError("project: " + builddir + " invalid status: " + p.status) xmlpath = os.path.join(builddir, "source.xml") xml = ElbeXML(xmlpath) if not new_version is None: xml.node("/project/version").set_text(new_version) xml.xml.write(xmlpath) p.version = xml.text("/project/version")
def run_command(argv): oparser = OptionParser( usage="usage: %prog add [options] <xmlfile> <pkg1> [pkgN]") (_, args) = oparser.parse_args(argv) if len(args) < 2: print("Wrong number of arguments") oparser.print_help() sys.exit(20) try: xml = ElbeXML(args[0]) except Exception as e: print("Error reading xml file: %s" % str(e)) sys.exit(20) for a in args[1:]: try: xml.add_target_package(a) except Exception as e: print("Error adding package %s: %s" % (a, str(e))) sys.exit(20) try: xml.xml.write(args[0]) except BaseException: print("Unable to write new xml file") sys.exit(20)
def extract_cdrom(cdrom): """ Extract cdrom iso image returns a TmpdirFilesystem() object containing the source.xml, which is also validated. """ tmp = TmpdirFilesystem() system('7z x -o%s "%s" source.xml' % (tmp.path, cdrom)) print("", file=sys.stderr) if not tmp.isfile('source.xml'): print("Iso image does not contain a source.xml file", file=sys.stderr) print("This is not supported by 'elbe initvm'", file=sys.stderr) print("", file=sys.stderr) print("Exiting !!!", file=sys.stderr) sys.exit(20) try: exml = ElbeXML(tmp.fname('source.xml'), url_validation=ValidationMode.NO_CHECK) except ValidationError as e: print("Iso image does contain a source.xml file.", file=sys.stderr) print("But that xml does not validate correctly", file=sys.stderr) print("", file=sys.stderr) print("Exiting !!!", file=sys.stderr) print(e) sys.exit(20) print("Iso Image with valid source.xml detected !") print("Image was generated using Elbe Version %s" % exml.get_elbe_version()) return tmp
def run_command(argv): oparser = OptionParser(usage="usage: %prog validate <xmlfile>") oparser.add_option("--validate-urls", dest="validate_urls", help="try to access specified repositories", default=False, action="store_true") (opt, args) = oparser.parse_args(argv) if len(args) < 1: oparser.print_help() sys.exit(20) if not os.path.exists(args[0]): print("%s - file not found" % args[0]) oparser.print_help() sys.exit(20) validation = validate_xml(args[0]) if validation: print("validation failed") for i in validation: print(i) sys.exit(20) if opt.validate_urls: try: ElbeXML(args[0], url_validation=ValidationMode.CHECK_ALL) except ValidationError as e: print(e) sys.exit(20) sys.exit(0)
def run_command(argv): oparser = OptionParser( usage="usage: %prog add [options] <xmlfile> <pkg1> [pkgN]") (_, args) = oparser.parse_args(argv) if len(args) < 2: print("Wrong number of arguments") oparser.print_help() sys.exit(20) xmlfile = args[0] pkg_lst = args[1:] try: xml = ElbeXML(xmlfile) except ValidationError as E: print("Error while reading xml file %s: %s" % (xmlfile, E)) sys.exit(20) for pkg in pkg_lst: try: xml.add_target_package(pkg) except ValueError as E: print("Error while adding package %s to %s: %s" % (pkg, xmlfile, E)) sys.exit(20) try: xml.xml.write(xmlfile) sys.exit(0) except PermissionError as E: print("Unable to truncate file %s: %s" % (xmlfile, E)) sys.exit(20)
def execute(self, client, _opt, args): if len(args) != 2: print( "usage: elbe control set_xml <project_dir> <xml>", file=sys.stderr) sys.exit(20) builddir = args[0] filename = args[1] try: x = ElbeXML( filename, skip_validate=True, url_validation=ValidationMode.NO_CHECK) except IOError: print("%s is not a valid elbe xml file" % filename) sys.exit(20) if not x.has('target'): print("<target> is missing, this file can't be built in an initvm", file=sys.stderr) sys.exit(20) size = 1024 * 1024 part = 0 with open(filename, "rb") as fp: while True: xml_base64 = binascii.b2a_base64(fp.read(size)).decode('ascii') # finish upload if len(xml_base64) == 1: part = client.service.upload_file(builddir, "source.xml", xml_base64, -1) else: part = client.service.upload_file(builddir, "source.xml", xml_base64, part) if part == -1: print("project busy, upload not allowed") return part if part == -2: print("upload of xml finished") return 0
def set_xml(self, xmlpath): # Use supplied XML file, if given, otherwise change to source.xml if not xmlpath: xmlpath = os.path.join(self.builddir, "source.xml") newxml = ElbeXML(xmlpath, buildtype=self.override_buildtype, skip_validate=self.skip_validate, url_validation=self.url_validation) # New XML file has to have the same architecture oldarch = self.xml.text("project/arch", key="arch") newarch = newxml.text("project/arch", key="arch") if newarch != oldarch: raise IncompatibleArchitectureException(oldarch, newarch) # Throw away old APT cache, targetfs and buildenv self.targetfs = None self.buildenv = None # dont create sysroot instance, it should be build from scratch # each time, because the pkglist including the -dev packages is # tracked nowhere. self.sysrootenv = None self.log.do('rm -rf %s' % self.sysrootpath) self.xml = newxml # Create a new BuildEnv instance, if we have a build directory if self.has_full_buildenv(): self.buildenv = BuildEnv(self.xml, self.log, self.chrootpath, clean=False) # Create TargetFs instance, if the target directory exists. # We use the old content of the directory if no rebuild is done, so # don't clean it (yet). if os.path.exists(self.targetpath): self.targetfs = TargetFs(self.targetpath, self.log, self.xml, clean=False) else: self.targetfs = None
def copy_initvmnode(self): source = self.xml source_path = "/var/cache/elbe/source.xml" try: initxml = ElbeXML(source_path, skip_validate=self.skip_validate, url_validation=ValidationMode.NO_CHECK) self.xml.get_initvmnode_from(initxml) except ValidationError: logging.exception( "%s validation failed. " "Will not copy initvm node", source_path) except IOError: logging.exception("%s not available. " "Can not copy initvm node", source_path) except NoInitvmNode: logging.exception( "%s is available. But it does not " "contain an initvm node", source_path)
def set_xml(self, builddir, xml_file): # This method can throw: ElbeDBError, ValidationError, OSError if not os.path.exists(builddir): raise ElbeDBError("project directory does not exist") srcxml_fname = os.path.join(builddir, "source.xml") if xml_file is None: xml_file = srcxml_fname with session_scope(self.session) as s: p = None try: p = s.query (Project). \ filter(Project.builddir == builddir).one() except NoResultFound: raise ElbeDBError( "project %s is not registered in the database" % builddir) if p.status == "busy": raise ElbeDBError( "cannot set XML file while project %s is busy" % builddir) xml = ElbeXML( xml_file, url_validation=ValidationMode.NO_CHECK) #ValidationError p.name = xml.text("project/name") p.version = xml.text("project/version") p.edit = datetime.utcnow() if p.status == "empty_project" or p.status == "build_failed": p.status = "needs_build" elif p.status == "build_done": p.status = "has_changes" if xml_file != srcxml_fname: copyfile(xml_file, srcxml_fname) #OSError self._update_project_file(s, builddir, "source.xml", "application/xml", "ELBE recipe of the project")
def execute(self, client, opt, args): if len(args) != 1: print("usage: elbe control create_project <xmlfile>", file=sys.stderr) sys.exit(20) filename = args[0] if not os.path.isfile(filename): print("%s doesn't exist" % filename, file=sys.stderr) sys.exit(20) x = ElbeXML(filename, skip_validate=True, skip_urlcheck=True) if not x.has('target'): print("<target> is missing, this file can't be built in an initvm", file=sys.stderr) sys.exit(20) with file(filename, "r") as fp: xml_base64 = binascii.b2a_base64(fp.read()) print(client.service.create_project(xml_base64))
def save_version(self, builddir, description=None): with session_scope(self.session) as s: try: p = s.query( Project ).filter( Project.builddir == builddir).\ one() except NoResultFound: raise ElbeDBError( "project %s is not registered in the database" % builddir) assert p.status == "busy" sourcexmlpath = os.path.join(builddir, "source.xml") sourcexml = ElbeXML(sourcexmlpath, url_validation=ValidationMode.NO_CHECK) version = sourcexml.text("project/version") if s.query( ProjectVersion ).\ filter( ProjectVersion.builddir == builddir ).\ filter( ProjectVersion.version == version ).count() > 0: raise ElbeDBError( "Version %s already exists for project in %s, " "please change version number first" %\ (version, builddir) ) versionxmlname = get_versioned_filename(p.name, version, ".version.xml") versionxmlpath = os.path.join(builddir, versionxmlname) copyfile(sourcexmlpath, versionxmlpath) v = ProjectVersion(builddir=builddir, version=version, description=description) s.add(v) self._update_project_file(s, builddir, versionxmlname, "application/xml", "source.xml for version %s" % version)
def set_xml(self, xmlpath): # Use supplied XML file, if given, otherwise change to source.xml if not xmlpath: xmlpath = os.path.join(self.builddir, "source.xml") newxml = ElbeXML(xmlpath, buildtype=self.override_buildtype, skip_validate=self.skip_validate, skip_urlcheck=self.skip_urlcheck) # New XML file has to have the same architecture oldarch = self.xml.text("project/arch", key="arch") newarch = newxml.text("project/arch", key="arch") if newarch != oldarch: raise IncompatibleArchitectureException(oldarch, newarch) # Throw away old APT cache, targetfs and buildenv self._rpcaptcache = None self.targetfs = None self.buildenv = None self.xml = newxml # Create a new BuildEnv instance, if we have a build directory if self.has_full_buildenv(): self.buildenv = BuildEnv(self.xml, self.log, self.chrootpath) # Create TargetFs instance, if the target directory exists. # We use the old content of the directory if no rebuild is done, so # don't clean it (yet). if os.path.exists(self.targetpath): self.targetfs = TargetFs(self.targetpath, self.log, self.buildenv.xml, clean=False) else: self.targetfs = None
def gen_update_pkg (project, xml_filename, upd_filename, override_buildtype = None, skip_validate = False, debug = False, cmd_dir = None, cfg_dir=None): if xml_filename: xml = ElbeXML( xml_filename, buildtype=override_buildtype, skip_validate=skip_validate ) if not xml.has("fullpkgs"): raise MissingData("Xml does not have fullpkgs list") if not project.xml.has("fullpkgs"): raise MissingData("Source Xml does not have fullpkgs list") if not project.buildenv.rfs: raise MissingData("Target does not have a build environment") cache = project.get_rpcaptcache() instpkgs = cache.get_installed_pkgs() instindex = {} for p in instpkgs: instindex[p.name] = p xmlpkgs = xml.node("/fullpkgs") xmlindex = {} fnamelist = [] for p in xmlpkgs: name = p.et.text ver = p.et.get('version') md5 = p.et.get('md5') xmlindex[name] = p if not name in instindex: print "package removed: " + name continue ipkg = instindex[name] comp = cache.compare_versions(ipkg.installed_version, ver) pfname = ipkg.installed_deb if comp == 0: print "package ok: " + name + "-" + ipkg.installed_version if debug: fnamelist.append( pfname ) continue if comp > 0: print "package upgrade: " + pfname fnamelist.append( pfname ) else: print "package downgrade: " + name + "-" + ipkg.installed_version for p in instpkgs: if p.name in xmlindex: continue print "package new installed " + p.name pfname = p.installed_deb fnamelist.append( pfname ) update = os.path.join(project.builddir, "update") if os.path.exists( update ): rmtree( update ) os.system( 'mkdir -p %s' % update ) if xml_filename: repodir = os.path.join(update, "repo" ) repo = UpdateRepo( xml, repodir, project.log ) for fname in fnamelist: path = os.path.join( project.chrootpath, "var/cache/apt/archives", fname ) repo.includedeb( path ) repo.finalize () dump_fullpkgs(project.xml, project.buildenv.rfs, cache) project.xml.xml.write( os.path.join( update, "new.xml" ) ) os.system( "cp %s %s" % (xml_filename, os.path.join( update, "base.xml" )) ) else: os.system( "cp source.xml update/new.xml") if project.presh_file: copyfile (project.presh_file, update + '/pre.sh') os.chmod (update + '/pre.sh', 0755) if project.postsh_file: copyfile (project.postsh_file, update + '/post.sh') os.chmod (update + '/post.sh', 0755) if cmd_dir: inlucdedir (update, 'cmd', cmd_dir, mode=0755) if cfg_dir: inlucdedir (update, 'conf', cfg_dir) create_zip_archive( upd_filename, update, "." ) if project.postbuild_file: project.log.h2 ("postbuild script") project.log.do (project.postbuild_file+' "%s %s %s"'%( upd_filename, project.xml.text ("project/version"), project.xml.text ("project/name")), allow_fail=True)
def install_packages(self, target, buildenv=False): # pylint: disable=too-many-statements # pylint: disable=too-many-branches with target: # First update the apt cache try: self.get_rpcaptcache(env=target).update() except Exception as e: raise AptCacheUpdateError(e) # Then dump the debootstrap packages if target.fresh_debootstrap: if target.need_dumpdebootstrap: dump_debootstrappkgs(self.xml, self.get_rpcaptcache(env=target)) dump_initvmpkgs(self.xml) target.need_dumpdebootstrap = False source = self.xml source_path = "/var/cache/elbe/source.xml" try: initxml = ElbeXML(source_path, skip_validate=self.skip_validate, url_validation=ValidationMode.NO_CHECK) self.xml.get_initvmnode_from(initxml) except ValidationError: logging.exception( "%s validation failed. " "Will not copy initvm node", source_path) except IOError: logging.exception( "%s not available. " "Can not copy initvm node", source_path) except NoInitvmNode: logging.exception( "%s is available. But it does not " "contain an initvm node", source_path) else: sourcepath = os.path.join(self.builddir, "source.xml") source = ElbeXML(sourcepath, buildtype=self.override_buildtype, skip_validate=self.skip_validate, url_validation=self.url_validation) self.xml.get_debootstrappkgs_from(source) try: self.xml.get_initvmnode_from(source) except NoInitvmNode: logging.exception("source.xml is available. " "But it does not contain an initvm node") # Seed /etc, we need /etc/hosts for hostname -f to work correctly if not buildenv: target.seed_etc() # remove all non-essential packages to ensure that on a incremental # build packages can be removed debootstrap_pkgs = [] for p in self.xml.node("debootstrappkgs"): debootstrap_pkgs.append(p.et.text) pkgs = target.xml.get_target_packages() + debootstrap_pkgs if buildenv: pkgs = pkgs + target.xml.get_buildenv_packages() # Now install requested packages for p in pkgs: try: self.get_rpcaptcache(env=target).mark_install(p, None) except KeyError: logging.exception("No Package %s", p) except SystemError: logging.exception( "Unable to correct problems " "in package %s", p) # temporary disabled because of # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=776057 # the functions cleans up to much # self.get_rpcaptcache().cleanup(debootstrap_pkgs + pkgs) try: self.get_rpcaptcache(env=target).commit() except SystemError as e: logging.exception("Commiting changes failed") raise AptCacheCommitError(str(e))
def __init__(self, builddir, xmlpath=None, name=None, override_buildtype=None, skip_validate=False, url_validation=ValidationMode.CHECK_ALL, rpcaptcache_notifier=None, private_data=None, postbuild_file=None, presh_file=None, postsh_file=None, savesh_file=None): # pylint: disable=too-many-arguments self.builddir = os.path.abspath(str(builddir)) self.chrootpath = os.path.join(self.builddir, "chroot") self.targetpath = os.path.join(self.builddir, "target") self.sysrootpath = os.path.join(self.builddir, "sysroot") self.sdkpath = os.path.join(self.builddir, "sdk") self.validationpath = os.path.join(self.builddir, "validation.txt") self.name = name self.override_buildtype = override_buildtype self.skip_validate = skip_validate self.url_validation = url_validation self.postbuild_file = postbuild_file self.presh_file = presh_file self.postsh_file = postsh_file self.savesh_file = savesh_file self.private_data = private_data # Apt-Cache will be created on demand with the specified notifier by # the get_rpcaptcache method self._rpcaptcache = None self.rpcaptcache_notifier = rpcaptcache_notifier # Initialise Repo Images to Empty list. self.repo_images = [] self.orig_fname = None self.orig_files = [] # Use supplied XML file, if given, otherwise use the source.xml # file of the project if xmlpath: self.xml = ElbeXML(xmlpath, buildtype=override_buildtype, skip_validate=skip_validate, url_validation=url_validation) else: sourcexmlpath = os.path.join(self.builddir, "source.xml") self.xml = ElbeXML(sourcexmlpath, buildtype=override_buildtype, skip_validate=skip_validate, url_validation=url_validation) self.arch = self.xml.text("project/arch", key="arch") self.codename = self.xml.text("project/suite") if not self.name: self.name = self.xml.text("project/name") self.repo = ProjectRepo(self.arch, self.codename, os.path.join(self.builddir, "repo")) # Create BuildEnv instance, if the chroot directory exists and # has an etc/elbe_version if os.path.exists(self.chrootpath): self.buildenv = BuildEnv(self.xml, self.chrootpath, clean=False) else: self.buildenv = None # Create TargetFs instance, if the target directory exists if os.path.exists(self.targetpath) and self.buildenv: self.targetfs = TargetFs(self.targetpath, self.buildenv.xml, clean=False) else: self.targetfs = None # don't create sysroot instance, it should be built from scratch # each time, because the pkglist including the -dev packages is # tracked nowhere. self.sysrootenv = None do('rm -rf %s' % self.sysrootpath) # same for host_sysroot instance recreate it in any case self.host_sysrootenv = None
def run_command(argv): oparser = OptionParser( usage="usage: %prog pkgdiff [options] <rfs1> <rfs2>") oparser.add_option("--noauto", action="store_true", dest="noauto", default=False, help="Dont compare automatically installed Packages") (opt, args) = oparser.parse_args(argv) if len(args) != 2: print "Wrong number of arguments" oparser.print_help() sys.exit(20) gen_rfs = args[0] fix_rfs = args[1] x = os.path.join(gen_rfs, 'etc/elbe_base.xml') xml = ElbeXML(x, skip_validate=True, skip_urlcheck=True) arch = xml.text('project/arch', key='arch') apt_pkg.init_config() apt_pkg.config.set('RootDir', gen_rfs) apt_pkg.config.set('APT::Architecture', arch) apt_pkg.init_system() gen_cache = apt_pkg.Cache(apt.progress.base.OpProgress()) gc = apt.Cache() gen_pkgs = {} for p in gen_cache.packages: if opt.noauto: if p.current_ver and not gc[ p.name].is_auto_installed and not p.essential: gen_pkgs[p.name] = p.current_ver else: if p.current_ver and not p.essential: gen_pkgs[p.name] = p.current_ver apt_pkg.init_config() apt_pkg.config.set('RootDir', fix_rfs) apt_pkg.config.set('APT::Architecture', arch) apt_pkg.init_system() fix_cache = apt_pkg.Cache(apt.progress.base.OpProgress()) fc = apt.Cache() fix_pkgs = {} for p in fix_cache.packages: if opt.noauto: if p.current_ver and not fc[ p.name].is_auto_installed and not p.essential: fix_pkgs[p.name] = p.current_ver else: if p.current_ver and not p.essential: fix_pkgs[p.name] = p.current_ver for p in fix_pkgs.keys(): if not p in gen_pkgs.keys(): print "+<pkg>%s</pkg>" % p for p in gen_pkgs.keys(): if not p in fix_pkgs.keys(): print "-<pkg>%s</pkg>" % p for p in fix_pkgs.keys(): if p in gen_pkgs.keys() and fix_pkgs[p] != gen_pkgs[p]: print "%s: Version mismatch %s != %s" % (p, fix_pkgs[p], gen_pkgs[p])
def run_command(argv): # TODO - Set threshold and remove pylint directives # # We might want to make the threshold higher for certain # files/directories or just globaly. # pylint: disable=too-many-locals # pylint: disable=too-many-branches # pylint: disable=too-many-statements oparser = OptionParser( usage="usage: %prog fetch_initvm_pkgs [options] <xmlfile>") oparser.add_option("-b", "--binrepo", dest="binrepo", default="/var/cache/elbe/initvm-bin-repo", help="directory where the bin repo should reside") oparser.add_option("-s", "--srcrepo", dest="srcrepo", default="/var/cache/elbe/initvm-src-repo", help="directory where the src repo should reside") oparser.add_option("--skip-validation", action="store_true", dest="skip_validation", default=False, help="Skip xml schema validation") oparser.add_option("--cdrom-mount-path", dest="cdrom_path", help="path where cdrom is mounted") oparser.add_option("--cdrom-device", dest="cdrom_device", help="cdrom device, in case it has to be mounted") oparser.add_option("--apt-archive", dest="archive", default="/var/cache/elbe/binaries/main", help="path where binary packages are downloaded to.") oparser.add_option("--src-archive", dest="srcarchive", default="/var/cache/elbe/sources", help="path where src packages are downloaded to.") oparser.add_option("--skip-build-sources", action="store_false", dest="build_sources", default=True, help="Skip downloading Source Packages") oparser.add_option("--skip-build-bin", action="store_false", dest="build_bin", default=True, help="Skip downloading binary packages") (opt, args) = oparser.parse_args(argv) if len(args) != 1: print("wrong number of arguments") oparser.print_help() sys.exit(20) try: xml = ElbeXML(args[0], skip_validate=opt.skip_validation) except ValidationError as e: print(str(e)) print("xml validation failed. Bailing out") sys.exit(20) with elbe_logging({"streams": sys.stdout}): if opt.cdrom_path: if opt.cdrom_device: do('mount "%s" "%s"' % (opt.cdrom_device, opt.cdrom_path)) # a cdrom build is identified by the cdrom option # the xml file that is copied into the initvm # by the initrd does not have the cdrom tags setup. mirror = "file://%s" % opt.cdrom_path else: mirror = xml.get_initvm_primary_mirror(opt.cdrom_path) init_codename = xml.get_initvm_codename() # Binary Repo # repo = CdromInitRepo(init_codename, opt.binrepo, mirror) hostfs.mkdir_p(opt.archive) if opt.build_bin: pkglist = get_initvm_pkglist() cache = Cache() cache.open() for pkg in pkglist: pkg_id = "%s-%s" % (pkg.name, pkg.installed_version) try: p = cache[pkg.name] pkgver = p.installed deb = fetch_binary(pkgver, opt.archive, ElbeAcquireProgress(cb=None)) repo.includedeb(deb, 'main') except ValueError: logging.exception('No package "%s"', pkg_id) except FetchError: logging.exception( 'Package "%s-%s" could not be downloaded', pkg.name, pkgver.version) except TypeError: logging.exception('Package "%s" missing name or version', pkg_id) repo.finalize() # Source Repo # repo = CdromSrcRepo(init_codename, init_codename, opt.srcrepo, 0, mirror) hostfs.mkdir_p(opt.srcarchive) # a cdrom build does not have sources # skip adding packages to the source repo # # FIXME: we need a way to add source cdroms later on if opt.cdrom_path: opt.build_sources = False if opt.build_sources: for pkg in pkglist: pkg_id = "%s-%s" % (pkg.name, pkg.installed_version) try: p = cache[pkg.name] pkgver = p.installed dsc = pkgver.fetch_source(opt.srcarchive, ElbeAcquireProgress(cb=None), unpack=False) repo.include_init_dsc(dsc, 'initvm') except ValueError: logging.exception('No package "%s"', pkg_id) except FetchError: logging.exception( 'Package "%s-%s" could not be downloaded', pkg.name, pkgver.version) except TypeError: logging.exception('Package "%s" missing name or version', pkg_id) repo.finalize() if opt.cdrom_device: do('umount "%s"' % opt.cdrom_device)
def install_packages(self, buildenv=False): with self.buildenv: # First update the apt cache try: self.get_rpcaptcache().update() except: self.log.printo("update cache failed") raise AptCacheUpdateError() # Then dump the debootstrap packages if self.buildenv.fresh_debootstrap: if self.buildenv.need_dumpdebootstrap: dump_debootstrappkgs(self.xml, self.get_rpcaptcache()) dump_initvmpkgs(self.xml) self.buildenv.need_dumpdebootstrap = False source = self.xml try: initxml = ElbeXML("/var/cache/elbe/source.xml", skip_validate=self.skip_validate, skip_urlcheck=True) self.xml.get_initvmnode_from(initxml) except ValidationError as e: self.log.printo( "/var/cache/elbe/source.xml validation failed") self.log.printo(str(e)) self.log.printo("will not copy initvm node") except IOError: self.log.printo("/var/cache/elbe/source.xml not available") self.log.printo("can not copy initvm node") except NoInitvmNode: self.log.printo("/var/cache/elbe/source.xml is available") self.log.printo("But it does not contain an initvm node") else: sourcepath = os.path.join(self.builddir, "source.xml") source = ElbeXML(sourcepath, buildtype=self.override_buildtype, skip_validate=self.skip_validate, skip_urlcheck=self.skip_urlcheck) self.xml.get_debootstrappkgs_from(source) try: self.xml.get_initvmnode_from(source) except NoInitvmNode: self.log.printo("source.xml is available") self.log.printo("But it does not contain an initvm node") # Seed /etc, we need /etc/hosts for hostname -f to work correctly if not buildenv: self.buildenv.seed_etc() # remove all non-essential packages to ensure that on a incremental # build packages can be removed debootstrap_pkgs = [] for p in self.xml.node("debootstrappkgs"): debootstrap_pkgs.append(p.et.text) pkgs = self.buildenv.xml.get_target_packages() if buildenv: pkgs = pkgs + self.buildenv.xml.get_buildenv_packages() # Now install requested packages for p in pkgs: try: self.get_rpcaptcache().mark_install(p, None) except KeyError: self.log.printo("No Package " + p) except SystemError: self.log.printo("Unable to correct problems " + p) # temporary disabled because of # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=776057 # the functions cleans up to much # self.get_rpcaptcache().cleanup(debootstrap_pkgs + pkgs) try: self.get_rpcaptcache().commit() except SystemError: self.log.printo("commiting changes failed") raise AptCacheCommitError()
def execute(self, initvmdir, opt, args): try: have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1") except CommandError as e: print( "tmux execution failed, tmux version 1.9 or higher is required" ) sys.exit(20) if have_session == 256: print("ElbeInitVMSession does not exist in tmux.", file=sys.stderr) print("Try 'elbe initvm start' to start the session.", file=sys.stderr) sys.exit(20) try: system('%s initvm ensure --directory "%s"' % (elbe_exe, initvmdir)) except CommandError: print("Starting the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) # Init cdrom to None, if we detect it, we set it cdrom = None if len(args) == 1: if args[0].endswith('.xml'): # We have an xml file, use that for elbe init xmlfile = args[0] elif args[0].endswith('.iso'): # We have an iso image, extract xml from there. tmp = TmpdirFilesystem() os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0])) print('', file=sys.stderr) if not tmp.isfile('source.xml'): print('Iso image does not contain a source.xml file', file=sys.stderr) print('This is not supported by "elbe initvm"', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) try: exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True) except ValidationError as e: print('Iso image does contain a source.xml file.', file=sys.stderr) print('But that xml does not validate correctly', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) print('Iso Image with valid source.xml detected !') print('Image was generated using Elbe Version %s' % exml.get_elbe_version()) xmlfile = tmp.fname('source.xml') cdrom = args[0] else: print('Unknown file ending (use either xml or iso)', file=sys.stderr) sys.exit(20) ret, prjdir, err = command_out_stderr( '%s control create_project --retries 60 "%s"' % (elbe_exe, xmlfile)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() if cdrom is not None: print("Uploading CDROM. This might take a while") try: system('%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom)) except CommandError: print("elbe control set_cdrom Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Upload finished") build_opts = '' if opt.build_bin: build_opts += '--build-bin ' if opt.build_sources: build_opts += '--build-sources ' try: system('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts)) except CommandError: print("elbe control build Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Build started, waiting till it finishes") try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Build finished !") print("") try: system('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir)) except CommandError: print("Project failed to generate validation.txt", file=sys.stderr) print("Getting log.txt", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("Failed to dump log.txt", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if opt.skip_download: print("") print("Listing available files:") print("") try: system('%s control get_files "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print('Get Files with: elbe control get_file "%s" <filename>' % prjdir) else: print("") print("Getting generated Files") print("") ensure_outdir(wdfs, opt) try: system('%s control get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if not opt.keep_files: try: system('%s control del_project "%s"' % (elbe_exe, prjdir)) except CommandError: print("remove project from initvm failed", file=sys.stderr) sys.exit(20)
def execute(self, initvmdir, opt, args): try: have_session = os.system( "tmux has-session -t ElbeInitVMSession >/dev/null 2>&1") except CommandError as e: print( "tmux execution failed, tmux version 1.9 or higher is required" ) sys.exit(20) if have_session == 0: print("ElbeInitVMSession already exists in tmux.", file=sys.stderr) print("", file=sys.stderr) print( "There can only exist a single ElbeInitVMSession, and this session", file=sys.stderr) print("can also be used to make your build.", file=sys.stderr) print( "See 'elbe initvm submit', 'elbe initvm attach' and 'elbe control'", file=sys.stderr) sys.exit(20) # Init cdrom to None, if we detect it, we set it cdrom = None if len(args) == 1: if args[0].endswith('.xml'): # We have an xml file, use that for elbe init exampl = args[0] try: xml = etree(exampl) except ValidationError as e: print('XML file is inavlid: ' + str(e)) # Use default XML if no initvm was specified if not xml.has("initvm"): exampl = os.path.join(elbepack.__path__[0], "init/default-init.xml") elif args[0].endswith('.iso'): # We have an iso image, extract xml from there. tmp = TmpdirFilesystem() os.system('7z x -o%s "%s" source.xml' % (tmp.path, args[0])) if not tmp.isfile('source.xml'): print('Iso image does not contain a source.xml file', file=sys.stderr) print('This is not supported by "elbe initvm"', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) try: exml = ElbeXML(tmp.fname('source.xml'), skip_urlcheck=True) except ValidationError as e: print('Iso image does contain a source.xml file.', file=sys.stderr) print('But that xml does not validate correctly', file=sys.stderr) print('', file=sys.stderr) print('Exiting !!!', file=sys.stderr) sys.exit(20) print('Iso Image with valid source.xml detected !') print('Image was generated using Elbe Version %s' % exml.get_elbe_version()) exampl = tmp.fname('source.xml') cdrom = args[0] else: print('Unknown file ending (use either xml or iso)', file=sys.stderr) sys.exit(20) else: # No xml File was specified, build the default elbe-init-with-ssh exampl = os.path.join(elbepack.__path__[0], "init/default-init.xml") try: if opt.devel: devel = ' --devel' else: devel = '' if cdrom: system('%s init %s --directory "%s" --cdrom "%s" "%s"' % (elbe_exe, devel, initvmdir, cdrom, exampl)) else: system('%s init %s --directory "%s" "%s"' % (elbe_exe, devel, initvmdir, exampl)) except CommandError: print("'elbe init' Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('cd "%s"; make' % (initvmdir)) except CommandError: print("Building the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s initvm start --directory "%s"' % (elbe_exe, initvmdir)) except CommandError: print("Starting the initvm Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if len(args) == 1: # if provided xml file has no initvm section exampl is set to a # default initvm XML file. But we need the original file here if args[0].endswith('.xml'): # stop here if no project node was specified try: x = ElbeXML(args[0]) except ValidationError as e: print('XML file is inavlid: ' + str(e)) sys.exit(20) if not x.has('project'): print( 'elbe initvm ready: use "elbe initvm submit myproject.xml" to build a project' ) sys.exit(0) ret, prjdir, err = command_out_stderr( '%s control create_project "%s"' % (elbe_exe, args[0])) else: ret, prjdir, err = command_out_stderr( '%s control create_project "%s"' % (elbe_exe, exampl)) if ret != 0: print("elbe control create_project failed.", file=sys.stderr) print(err, file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) prjdir = prjdir.strip() if cdrom is not None: print("Uploading CDROM. This might take a while") try: system('%s control set_cdrom "%s" "%s"' % (elbe_exe, prjdir, cdrom)) except CommandError: print("elbe control set_cdrom Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("Upload finished") build_opts = '' if opt.build_bin: build_opts += '--build-bin ' if opt.build_sources: build_opts += '--build-sources ' try: system('%s control build "%s" %s' % (elbe_exe, prjdir, build_opts)) except CommandError: print("elbe control build Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) try: system('%s control wait_busy "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control wait_busy Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print("Build finished !") print("") try: system('%s control dump_file "%s" validation.txt' % (elbe_exe, prjdir)) except CommandError: print("Project failed to generate validation.txt", file=sys.stderr) print("Getting log.txt", file=sys.stderr) try: system('%s control dump_file "%s" log.txt' % (elbe_exe, prjdir)) except CommandError: print("Failed to dump log.txt", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) if opt.skip_download: print("") print("Listing available files:") print("") try: system('%s control get_files "%s"' % (elbe_exe, prjdir)) except CommandError: print("elbe control Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20) print("") print('Get Files with: elbe control get_file "%s" <filename>' % prjdir) else: ensure_outdir(wdfs, opt) try: system('%s control get_files --output "%s" "%s"' % (elbe_exe, opt.outdir, prjdir)) except CommandError: print("elbe control get_files Failed", file=sys.stderr) print("Giving up", file=sys.stderr) sys.exit(20)
def __init__(self, builddir, xmlpath=None, logpath=None, name=None, override_buildtype=None, skip_validate=False, skip_urlcheck=False, rpcaptcache_notifier=None, private_data=None, postbuild_file=None, presh_file=None, postsh_file=None, savesh_file=None): self.builddir = os.path.abspath(str(builddir)) self.chrootpath = os.path.join(self.builddir, "chroot") self.targetpath = os.path.join(self.builddir, "target") self.name = name self.override_buildtype = override_buildtype self.skip_validate = skip_validate self.skip_urlcheck = skip_urlcheck self.postbuild_file = postbuild_file self.presh_file = presh_file self.postsh_file = postsh_file self.savesh_file = savesh_file self.private_data = private_data # Apt-Cache will be created on demand with the specified notifier by # the get_rpcaptcache method self._rpcaptcache = None self.rpcaptcache_notifier = rpcaptcache_notifier # Initialise Repo Images to Empty list. self.repo_images = [] # Use supplied XML file, if given, otherwise use the source.xml # file of the project if xmlpath: self.xml = ElbeXML(xmlpath, buildtype=override_buildtype, skip_validate=skip_validate, skip_urlcheck=skip_urlcheck) else: sourcexmlpath = os.path.join(self.builddir, "source.xml") self.xml = ElbeXML(sourcexmlpath, buildtype=override_buildtype, skip_validate=skip_validate, skip_urlcheck=skip_urlcheck) # If logpath is given, use an AsciiDocLog instance, otherwise log # to stdout if logpath: self.log = ASCIIDocLog(logpath) else: self.log = StdoutLog() # Create BuildEnv instance, if the chroot directory exists and # has an etc/elbe_version if self.has_full_buildenv(): self.buildenv = BuildEnv(self.xml, self.log, self.chrootpath) else: self.buildenv = None self.targetfs = None return # Create TargetFs instance, if the target directory exists if os.path.exists(self.targetpath): self.targetfs = TargetFs(self.targetpath, self.log, self.buildenv.xml, clean=False) else: self.targetfs = None
def run_command(argv): # pylint: disable=too-many-locals # pylint: disable=too-many-statements # pylint: disable=too-many-branches oparser = OptionParser( usage="usage: %prog check_updates [options] <source-xmlfile>") oparser.add_option( "-s", "--script", dest="script", help="filename of script to run when an update is required") oparser.add_option("--skip-validation", action="store_true", dest="skip_validation", default=False, help="Skip xml schema validation") oparser.add_option("-c", "--changelogs", dest="changelogs", help="filename of changelog xml file") (opt, args) = oparser.parse_args(argv) if len(args) != 1: print("Wrong number of arguments") oparser.print_help() sys.exit(20) if not opt.skip_validation: validation = validate_xml(args[0]) if validation: print("xml validation failed. Bailing out") for i in validation: print(i) sys.exit(20) print("checking %s" % args[0]) xml = ElbeXML(args[0]) fullp = xml.node("fullpkgs") arch = xml.text("project/buildimage/arch", key="arch") v = virtapt.VirtApt(xml) for p in fullp: pname = p.et.text pauto = p.et.get('auto') if pauto != "true": v.mark_install(pname) errors = 0 required_updates = 0 update_packages = [] for p in fullp: xp = XMLPackage(p, arch) pname = p.et.text pauto = p.et.get('auto') if not v.has_pkg(xp.name): if not xp.is_auto_installed: print( "%s does not exist in cache but is specified in pkg-list" % xp.name) errors += 1 else: print("%s is no more required" % xp.name) required_updates += 1 continue if v.marked_install(xp.name): cver = v.get_candidate_ver(xp.name) if xp.installed_version != cver: print("%s: %s != %s" % (xp.name, xp.installed_version, cver)) required_updates += 1 if opt.changelogs: v.mark_pkg_download(xp.name) xp.candidate_version = cver update_packages.append(xp) sys.stdout.flush() sys.stderr.flush() if errors > 0: print("%d Errors occured, xml files needs fixing" % errors) if opt.script: system("%s ERRORS %s" % (opt.script, args[0]), allow_fail=True) elif required_updates > 0: print("%d updates required" % required_updates) if opt.changelogs: build_changelog_xml(v, opt, update_packages) if opt.script: system("%s UPDATE %s" % (opt.script, args[0]), allow_fail=True) else: print("No Updates available")