def getFile( url, destdir , filename='' ): """download file from 'url' into 'destdir'""" EmergeDebug.debug("getFile called. url: %s" % url, 1) if url == "": EmergeDebug.error("fetch: no url given") return False if UtilsCache.findApplication("wget"): return wgetFile( url, destdir , filename ) if not filename: _, _, path, _, _, _ = urllib.parse.urlparse( url ) filename = os.path.basename( path ) if os.path.exists(os.path.join( destdir, filename )): return True width, _ = shutil.get_terminal_size((80,20)) def dlProgress(count, blockSize, totalSize): percent = int(count * blockSize * 100 / totalSize) times = int((width - 20)/100 * percent) sys.stdout.write(("\r%s%3d%%" % ("#" * times, percent))) sys.stdout.flush() urllib.request.urlretrieve(url, filename = os.path.join( destdir, filename ), reporthook= dlProgress if EmergeDebug.verbose() >= 0 else None ) if EmergeDebug.verbose()>=0: sys.stdout.write("\n") sys.stdout.flush() return True
def getFile( url, destdir , filename='' ): """download file from 'url' into 'destdir'""" EmergeDebug.debug("getFile called. url: %s" % url, 1) if url == "": EmergeDebug.error("fetch: no url given") return False wgetpath = WGetExecutable if ( os.path.exists( wgetpath ) ): return wgetFile( url, destdir , filename ) scheme, host, path, _, _, _ = urllib.parse.urlparse( url ) filename = os.path.basename( path ) EmergeDebug.debug("%s\n%s\n%s\n%s" % (scheme, host, path, filename)) if ( scheme == "http" ): return getHttpFile( host, path, destdir, filename ) elif ( scheme == "ftp" ): return getFtpFile( host, path, destdir, filename ) else: EmergeDebug.error("getFile: protocol not understood") return False
def checkDigest(self): EmergeDebug.debug("ArchiveSource.checkDigest called", 2) filenames = self.localFileNames() if self.subinfo.hasTargetDigestUrls(): EmergeDebug.debug("check digests urls", 1) if not EmergeHash.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames): EmergeDebug.error("invalid digest file") return False elif self.subinfo.hasTargetDigests(): EmergeDebug.debug("check digests", 1) digests, algorithm = self.subinfo.targetDigest() if not EmergeHash.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames, digests, algorithm): EmergeDebug.error("invalid digest file") return False else: EmergeDebug.debug("print source file digests", 1) EmergeHash.printFilesDigests( EmergeStandardDirs.downloadDir(), filenames, self.subinfo.buildTarget, algorithm=EmergeHash.HashAlgorithm.SHA256) return True
def getPackagesCategories(packageName, defaultCategory = None): EmergeDebug.trace("getPackagesCategories for package name %s" % packageName) if defaultCategory is None: defaultCategory = emergeSettings.get("General","EMERGE_DEFAULTCATEGORY","kde") packageList, categoryList = [], [] if len( packageName.split( "/" ) ) == 1: if PortageInstance.isCategory( packageName ): EmergeDebug.debug("isCategory=True", 2) packageList = PortageInstance.getAllPackages( packageName ) categoryList = [ packageName ] * len(packageList) else: EmergeDebug.debug("isCategory=False", 2) if PortageInstance.isCategory( defaultCategory ) and PortageInstance.isPackage( defaultCategory, packageName ): # prefer the default category packageList = [ packageName ] categoryList = [ defaultCategory ] else: if PortageInstance.getCategory( packageName ): packageList = [ packageName ] categoryList = [ PortageInstance.getCategory( packageName ) ] elif len( packageName.split( "/" ) ) == 2: [ cat, pac ] = packageName.split( "/" ) if PortageInstance.isCategory( cat ): categoryList = [ cat ] else: return packageList, categoryList if len( categoryList ) > 0 and PortageInstance.isPackage( categoryList[0], pac ): packageList = [ pac ] if len( categoryList ) and len( packageList ): EmergeDebug.debug("added package %s/%s" % (categoryList[0], pac), 2) else: EmergeDebug.error("unknown packageName") return packageList, categoryList
def checkFilesDigests(downloaddir, filenames, digests=None, digestAlgorithm=HashAlgorithm.SHA1): """check digest of (multiple) files specified by 'filenames' from 'downloaddir'""" if type(digests) == list: digestList = digests else: digestList = [digests] for digests, filename in zip(digestList, filenames): EmergeDebug.debug("checking digest of: %s" % filename, 1) pathName = os.path.join(downloaddir, filename) if digests == None: for digestAlgorithm, digestFileEnding in HashAlgorithm.fileEndings().items(): digestFileName = pathName + digestFileEnding if not os.path.exists(digestFileName): digestFileName, _ = os.path.splitext(pathName) digestFileName += digestFileEnding if not os.path.exists(digestFileName): continue currentHash = digestFile(pathName, digestAlgorithm) with open(digestFileName, "rt+") as f: data = f.read() if not re.findall(currentHash, data): EmergeDebug.error("%s hash for file %s (%s) does not match (%s)" % ( digestAlgorithm.name, pathName, currentHash, data)) return False # digest provided in digests parameter else: currentHash = digestFile(pathName, digestAlgorithm) if len(digests) != len(currentHash) or digests.find(currentHash) == -1: EmergeDebug.error("%s hash for file %s (%s) does not match (%s)" % ( digestAlgorithm.name, pathName, currentHash, digests)) return False return True
def getVCRedistLocation(self, compiler): _file = None if compiler.isMSVC2015(): if compiler.isX64(): _file = os.path.join( self.getVCRuntimeLibrariesLocation(), "1033", "vcredist_x64.exe" ) elif compiler.isX86(): _file = os.path.join( self.getVCRuntimeLibrariesLocation(), "1033", "vcredist_x86.exe" ) if not os.path.isfile(_file): _file = None EmergeDebug.new_line() EmergeDebug.error("Assuming we can't find a c++ redistributable because the user hasn't got one. Must be fixed manually.") return _file
def __parse(self): try: _listfile = open(self.filename, 'r') except: EmergeDebug.error("couldn't open listfile") return for line in _listfile: line = line.strip() if line.startswith('#'): continue _cat, _pac, _target, _patch = line.split(",") self.packageList.append((_cat, _pac, _target, _patch)) self.isParsed = True
def runAction( self, command ): """ \todo TODO: rename the internal functions into the form cmdFetch, cmdCheckDigest etc then we get by without this dict: ok = getattr(self, 'cmd' + command.capitalize()() next we could """ functions = {"fetch": "fetch", "cleanimage": "cleanImage", "cleanbuild": "cleanBuild", "unpack": "unpack", "compile": "compile", "configure": "configure", "make": "make", "install": "install", "test": "unittest", "qmerge": "qmerge", "unmerge": "unmerge", "package": "createPackage", "createpatch": "createPatch", "geturls": "getUrls", "print-revision": "printSourceVersion", "print-files": "printFiles", "checkdigest": "checkDigest", "dumpdeps": "dumpDependencies"} if command in functions: try: ok = getattr(self, functions[command])() except AttributeError as e: raise portage.PortageException( str( e ), self.category, self.package, e ) else: ok = EmergeDebug.error( "command %s not understood" % command ) return ok
def getVCRedistLocation(self, compiler): _file = None if compiler.isMSVC2015(): if compiler.isX64(): _file = os.path.join(self.getVCRuntimeLibrariesLocation(), "1033", "vcredist_x64.exe") elif compiler.isX86(): _file = os.path.join(self.getVCRuntimeLibrariesLocation(), "1033", "vcredist_x86.exe") if not os.path.isfile(_file): _file = None EmergeDebug.new_line() EmergeDebug.error( "Assuming we can't find a c++ redistributable because the user hasn't got one. Must be fixed manually." ) return _file
def runAction( self, command ): """ \todo TODO: rename the internal functions into the form cmdFetch, cmdCheckDigest etc then we get by without this dict: ok = getattr(self, 'cmd' + command.capitalize()() next we could """ functions = {"fetch": "fetch", "cleanimage": "cleanImage", "cleanbuild": "cleanBuild", "unpack": "unpack", "compile": "compile", "configure": "configure", "make": "make", "install": "install", "test": "unittest", "qmerge": "qmerge", "unmerge": "unmerge", "package": "createPackage", "createpatch": "createPatch", "geturls": "getUrls", "print-revision": "printSourceVersion", "print-files": "printFiles", "checkdigest": "checkDigest", "dumpdeps": "dumpDependencies", "fetch-binary": "fetchBinary"} if command in functions: try: ok = getattr(self, functions[command])() except AttributeError as e: raise portage.PortageException( str( e ), self.category, self.package, e ) else: ok = EmergeDebug.error( "command %s not understood" % command ) return ok
def main(): rest, args = parseOptions() if args.type == "xml": output_type = OUTPUT_XML elif args.type == "kwi": output_type = OUTPUT_KWI else: output_type = OUTPUT_DOT depstyle = args.depstyle if not args.depstyle in ['both', 'runtime']: depstyle = "both" output = "" if hasattr(args, "filenames") and args.filenames != None: packageList = parsePackageListFiles(args.filenames) output = dumpDependenciesForPackageList(packageList, output_type, depstyle) elif rest: output = dumpDependencies(rest[0], output_type, depstyle) else: EmergeDebug.error("missing package list file or package/category") sys.exit(1) if hasattr(args, "outputname") and args.outputname: print("writing file ", args.outputname, os.path.dirname(args.outputname)) if os.path.dirname(args.outputname) and not os.path.exists( os.path.dirname(args.outputname)): os.makedirs(os.path.dirname(args.outputname)) with open(args.outputname, "w") as f: f.write(output) if output_type == OUTPUT_DOT: _graphviz = graphviz.GraphViz() if not _graphviz.runDot(args.outputname, args.outputname + '.pdf', 'pdf'): exit(1) # we don't want to open the output automatically, at least not always # _graphviz.openOutput() else: print(output)
def handlePackage( category, packageName, buildAction, continueFlag, skipUpToDateVcs ): EmergeDebug.debug_line() EmergeDebug.info("Handling package: %s, action: %s" % (packageName, buildAction)) success = True package = portage.getPackageInstance( category, packageName ) if package is None: raise portage.PortageException( "Package not found", category, packageName ) if buildAction in [ "all", "full-package", "update", "update-all" ]: success = success and doExec( package, "fetch", continueFlag ) if success and skipUpToDateVcs and package.subinfo.hasSvnTarget( ): revision = package.sourceVersion( ) for p in InstallDB.installdb.getInstalledPackages( category, packageName ): if p.getRevision( ) == revision: EmergeDebug.info("Skipping further actions, package is up-to-date") return True success = success and doExec( package, "unpack", continueFlag ) success = success and doExec( package, "compile" ) success = success and doExec( package, "cleanimage" ) success = success and doExec( package, "install" ) if buildAction in [ "all", "update", "update-all" ]: success = success and doExec( package, "qmerge" ) if buildAction == "full-package": success = success and doExec( package, "package" ) success = success or continueFlag elif buildAction in [ "fetch", "unpack", "configure", "compile", "make", "checkdigest", "dumpdeps", "test", "package", "unmerge", "cleanimage", "cleanbuild", "createpatch", "geturls", "print-revision", "print-files" ]: success = doExec( package, buildAction, continueFlag ) elif buildAction == "install": success = doExec( package, "cleanimage" ) success = success and doExec( package, "install", continueFlag ) elif buildAction == "qmerge": #success = doExec( package, "cleanimage" ) #success = success and doExec( package, "install") success = success and doExec( package, "qmerge" ) elif buildAction == "generate-jenkins-job": success = jenkins.generateJob(package) elif buildAction == "print-source-version": print( "%s-%s" % ( packageName, package.sourceVersion( ) ) ) success = True elif buildAction == "print-package-version": print( "%s-%s-%s" % ( packageName, compiler.getCompilerName( ), package.sourceVersion( ) ) ) success = True elif buildAction == "print-targets": portage.printTargets( category, packageName ) success = True else: success = EmergeDebug.error("could not understand this buildAction: %s" % buildAction) return success
def checkDigest(self): EmergeDebug.debug("ArchiveSource.checkDigest called", 2) filenames = self.localFileNames() if self.subinfo.hasTargetDigestUrls(): EmergeDebug.debug("check digests urls", 1) if not EmergeHash.checkFilesDigests(EmergeStandardDirs.downloadDir(), filenames): EmergeDebug.error("invalid digest file") return False elif self.subinfo.hasTargetDigests(): EmergeDebug.debug("check digests", 1) digests, algorithm = self.subinfo.targetDigest() if not EmergeHash.checkFilesDigests( EmergeStandardDirs.downloadDir(), filenames, digests, algorithm): EmergeDebug.error("invalid digest file") return False else: EmergeDebug.debug("print source file digests", 1) EmergeHash.printFilesDigests(EmergeStandardDirs.downloadDir(), filenames, self.subinfo.buildTarget, algorithm = EmergeHash.HashAlgorithm.SHA256) return True
def checkFilesDigests(downloaddir, filenames, digests=None, digestAlgorithm=HashAlgorithm.SHA1): """check digest of (multiple) files specified by 'filenames' from 'downloaddir'""" if type(digests) == list: digestList = digests else: digestList = [digests] for digests, filename in zip(digestList, filenames): EmergeDebug.debug("checking digest of: %s" % filename, 1) pathName = os.path.join(downloaddir, filename) if digests == None: for digestAlgorithm, digestFileEnding in HashAlgorithm.fileEndings( ).items(): digestFileName = pathName + digestFileEnding if not os.path.exists(digestFileName): digestFileName, _ = os.path.splitext(pathName) digestFileName += digestFileEnding if not os.path.exists(digestFileName): continue currentHash = digestFile(pathName, digestAlgorithm) with open(digestFileName, "rt+") as f: data = f.read() if not re.findall(currentHash, data): EmergeDebug.error( "%s hash for file %s (%s) does not match (%s)" % (digestAlgorithm.name, pathName, currentHash, data)) return False # digest provided in digests parameter else: currentHash = digestFile(pathName, digestAlgorithm) if len(digests) != len(currentHash) or digests.find( currentHash) == -1: EmergeDebug.error( "%s hash for file %s (%s) does not match (%s)" % (digestAlgorithm.name, pathName, currentHash, digests)) return False return True
def main(): rest, args = parseOptions() if args.type == "xml": output_type = OUTPUT_XML elif args.type == "kwi": output_type = OUTPUT_KWI else: output_type = OUTPUT_DOT depstyle = args.depstyle if not args.depstyle in ['both', 'runtime']: depstyle = "both" output = "" if hasattr(args, "filenames") and args.filenames != None: packageList = parsePackageListFiles( args.filenames ) output = dumpDependenciesForPackageList(packageList, output_type, depstyle) elif rest: output = dumpDependencies(rest[0], output_type, depstyle) else: EmergeDebug.error("missing package list file or package/category") sys.exit(1) if hasattr(args, "outputname") and args.outputname: print("writing file ", args.outputname, os.path.dirname( args.outputname )) if os.path.dirname( args.outputname ) and not os.path.exists( os.path.dirname( args.outputname ) ): os.makedirs( os.path.dirname( args.outputname ) ) with open(args.outputname, "w") as f: f.write( output ) if output_type == OUTPUT_DOT: _graphviz = graphviz.GraphViz() if not _graphviz.runDot( args.outputname, args.outputname + '.pdf', 'pdf' ): exit( 1 ) # we don't want to open the output automatically, at least not always # _graphviz.openOutput() else: print(output)
def unpackFile( downloaddir, filename, workdir ): """unpack file specified by 'filename' from 'downloaddir' into 'workdir'""" ( shortname, ext ) = os.path.splitext( filename ) if ( ext == ".zip" ): return unZip( os.path.join( downloaddir, filename ), workdir ) elif ( ext == ".7z" ): return un7zip( os.path.join( downloaddir, filename ), workdir, ext ) elif ( ext == ".tgz" ): return unTar( os.path.join( downloaddir, filename ), workdir ) elif ( ext == ".gz" or ext == ".bz2" or ext == ".lzma" or ext == ".xz" ): _, myext = os.path.splitext( shortname ) if ( myext == ".tar" ): return unTar( os.path.join( downloaddir, filename ), workdir ) else: EmergeDebug.error("unpacking %s" % myext) return False elif ( ext == ".exe" ): EmergeDebug.warning("unpack ignoring exe file") return True else: EmergeDebug.error("dont know how to unpack this file: %s" % filename) return False
def unZip( fileName, destdir ): """unzip file specified by 'file' into 'destdir'""" EmergeDebug.debug("unZip called: file %s to destination %s" % (fileName, destdir), 1) if not os.path.exists( destdir ): os.makedirs( destdir ) try: zipObj = zipfile.ZipFile( fileName ) except (zipfile.BadZipfile, IOError): EmergeDebug.error("couldn't extract file %s" % fileName) return False for name in zipObj.namelist(): if not name.endswith( '/' ): dirname = os.path.join( destdir, os.path.dirname( name ) ) if not os.path.exists( dirname ): os.makedirs( dirname ) with open( os.path.join( destdir, name ), 'wb' ) as outfile: outfile.write( zipObj.read( name ) ) return True
def unTar( fileName, destdir ): """unpack tar file specified by 'file' into 'destdir'""" EmergeDebug.debug("unTar called. file: %s, destdir: %s" % (fileName, destdir), 1) ( shortname, ext ) = os.path.splitext( fileName ) emerge_tmp = os.path.join(destdir,"emerge_tmp") mode = "r" if ( ext == ".gz" ): mode = "r:gz" #elif(ext == ".bz2"): #mode = "r:bz2" elif(ext == ".lzma" or ext == ".xz" or ext == ".bz2"): un7zip( fileName, emerge_tmp ) _, tarname = os.path.split( shortname ) fileName = os.path.join( emerge_tmp , tarname ) if not os.path.exists( fileName ): EmergeDebug.error("couldn't find file %s" % fileName) return False try: with tarfile.open( fileName, mode ) as tar: # FIXME how to handle errors here ? for tarMember in tar: try: if tarMember.issym(): tarDir = os.path.dirname(tarMember.name) target = tarMember.linkname if not target.startswith("/"):#abspath? target = os.path.normpath("%s/%s"%(tarDir, target)).replace("\\","/") if target in tar.getnames(): tar.extract(target, emerge_tmp ) shutil.move(os.path.join( emerge_tmp , tarDir , tarMember.linkname ),os.path.join( destdir , tarMember.name )) EmergeDebug.warning("Resolved symlink %s in tarfile %s to %s" % (tarMember.name, fileName , tarMember.linkname)) else: EmergeDebug.warning("link target %s for %s not included in tarfile" % (target , tarMember.name)) else: tar.extract(tarMember, destdir ) except tarfile.TarError: EmergeDebug.error("couldn't extract file %s to directory %s" % (fileName, destdir)) return False except IOError: EmergeDebug.warning("Failed to extract %s to directory %s" % (tarMember.name, destdir)) return True except tarfile.TarError as e: EmergeDebug.error("could not open existing tar archive: %s error: %s" % (fileName, e)) return False finally: if os.path.exists(emerge_tmp): shutil.rmtree(emerge_tmp)
def handlePackage(category, packageName, buildAction, continueFlag, skipUpToDateVcs): with EmergeTimer.Timer("HandlePackage %s/%s" % (category, packageName), 3) as timer: EmergeDebug.debug_line() EmergeDebug.info("Handling package: %s, action: %s" % (packageName, buildAction)) success = True package = portage.getPackageInstance(category, packageName) if package is None: raise portage.PortageException("Package not found", category, packageName) if buildAction in ["all", "full-package", "update", "update-all"]: success = success and doExec(package, "fetch", continueFlag) skip = False if success and skipUpToDateVcs and package.subinfo.hasSvnTarget(): revision = package.sourceVersion() for p in InstallDB.installdb.getInstalledPackages( category, packageName): if p.getRevision() == revision: EmergeDebug.info( "Skipping further actions, package is up-to-date") skip = True if not skip: success = success and doExec(package, "unpack", continueFlag) success = success and doExec(package, "compile") success = success and doExec(package, "cleanimage") success = success and doExec(package, "install") if buildAction in ["all", "update", "update-all"]: success = success and doExec(package, "qmerge") if buildAction == "full-package": success = success and doExec(package, "package") success = success or continueFlag elif buildAction in [ "fetch", "unpack", "configure", "compile", "make", "checkdigest", "dumpdeps", "test", "package", "unmerge", "cleanimage", "cleanbuild", "createpatch", "geturls", "print-revision", "print-files" ]: success = doExec(package, buildAction, continueFlag) elif buildAction == "install": success = doExec(package, "cleanimage") success = success and doExec(package, "install", continueFlag) elif buildAction == "qmerge": #success = doExec( package, "cleanimage" ) #success = success and doExec( package, "install") success = success and doExec(package, "qmerge") elif buildAction == "print-source-version": print("%s-%s" % (packageName, package.sourceVersion())) success = True elif buildAction == "print-package-version": print("%s-%s-%s" % (packageName, compiler.getCompilerName(), package.sourceVersion())) success = True elif buildAction == "print-targets": portage.printTargets(category, packageName) success = True else: success = EmergeDebug.error( "could not understand this buildAction: %s" % buildAction) timer.stop() utils.notify( "Emerge %s %s" % (buildAction, "succeeded" if success else "failed"), "%s of %s/%s %s after %s" % (buildAction, category, packageName, "succeeded" if success else "failed", timer), buildAction) return success
def handleSinglePackage(packageName, action, args): deplist = [] packageList = [] originalPackageList = [] categoryList = [] targetDict = dict() if action == "update-all": installedPackages = portage.PortageInstance.getInstallables() if portage.PortageInstance.isCategory(packageName): EmergeDebug.debug( "Updating installed packages from category " + packageName, 1) else: EmergeDebug.debug("Updating all installed packages", 1) packageList = [] for mainCategory, mainPackage in installedPackages: if portage.PortageInstance.isCategory(packageName) and ( mainCategory != packageName): continue if InstallDB.installdb.isInstalled( mainCategory, mainPackage, args.buildType ) \ and portage.isPackageUpdateable( mainCategory, mainPackage ): categoryList.append(mainCategory) packageList.append(mainPackage) EmergeDebug.debug("Will update packages: " + str(packageList), 1) elif args.list_file: listFileObject = open(args.list_file, 'r') for line in listFileObject: if line.strip().startswith('#'): continue try: cat, pac, tar, _ = line.split(',') except: continue categoryList.append(cat) packageList.append(pac) originalPackageList.append(pac) targetDict[cat + "/" + pac] = tar elif packageName: packageList, categoryList = portage.getPackagesCategories(packageName) for entry in packageList: EmergeDebug.debug("Checking dependencies for: %s" % entry, 1) for mainCategory, entry in zip(categoryList, packageList): deplist = portage.solveDependencies(mainCategory, entry, deplist, args.dependencyType, maxDepth=args.dependencydepth) # no package found if len(deplist) == 0: category = "" if not packageName.find("/") == -1: (category, package) = packageName.split("/") portageSearch.printSearch(category, packageName) return False for item in deplist: item.enabled = args.ignoreAllInstalled if args.ignoreInstalled and item.category in categoryList and item.package in packageList or packageIsOutdated( item.category, item.package): item.enabled = True if item.category + "/" + item.package in targetDict: item.target = targetDict[item.category + "/" + item.package] if args.target in list( portage.PortageInstance.getAllTargets(item.category, item.package).keys()): # if no target or a wrong one is defined, simply set the default target here item.target = args.target EmergeDebug.debug("dependency: %s" % item, 1) if not deplist: EmergeDebug.debug("<none>", 1) EmergeDebug.debug_line(1) #for item in deplist: # cat = item[ 0 ] # pac = item[ 1 ] # ver = item[ 2 ] # if portage.isInstalled( cat, pac, ver, buildType) and updateAll and not portage.isPackageUpdateable( cat, pac, ver ): # print "remove:", cat, pac, ver # deplist.remove( item ) if action == "install-deps": # the first dependency is the package itself - ignore it # TODO: why are we our own dependency? del deplist[0] elif action == "update-direct-deps": for item in deplist: item.enabled = True deplist.reverse() # package[0] -> category # package[1] -> package # package[2] -> version info = deplist[-1] if not portage.PortageInstance.isVirtualPackage( info.category, info.package ) and \ not action in [ "all", "install-deps"] and\ not args.list_file or\ action in ["print-targets"]:#not all commands should be executed on the deps if we are a virtual packages # if a buildAction is given, then do not try to build dependencies # and do the action although the package might already be installed. # This is still a bit problematic since packageName might not be a valid # package # for list files, we also want to handle fetching & packaging per package if not handlePackage(info.category, info.package, action, args.doContinue, args.update_fast): return False else: if args.dumpDepsFile: dumpDepsFileObject = open(args.dumpDepsFile, 'w+') dumpDepsFileObject.write("# dependency dump of package %s\n" % (packageName)) for info in deplist: isVCSTarget = False if args.dumpDepsFile: dumpDepsFileObject.write( ",".join([info.category, info.package, info.target, ""]) + "\n") isLastPackage = info == deplist[-1] if args.outDateVCS or (args.outDatePackage and isLastPackage): isVCSTarget = portage.PortageInstance.getUpdatableVCSTargets( info.category, info.package) != [] isInstalled = InstallDB.installdb.isInstalled( info.category, info.package) if args.list_file and action != "all": info.enabled = info.package in originalPackageList if (isInstalled and not info.enabled) and not ( isInstalled and (args.outDateVCS or (args.outDatePackage and isLastPackage)) and isVCSTarget): if EmergeDebug.verbose() > 1 and info.package == packageName: EmergeDebug.warning("already installed %s/%s" % (info.category, info.package)) elif EmergeDebug.verbose( ) > 2 and not info.package == packageName: EmergeDebug.warning("already installed %s/%s" % (info.category, info.package)) else: # in case we only want to see which packages are still to be build, simply return the package name if args.probe: EmergeDebug.warning("pretending %s" % info) else: if action in ["install-deps", "update-direct-deps"]: action = "all" if not handlePackage(info.category, info.package, action, args.doContinue, args.update_fast): EmergeDebug.error("fatal error: package %s/%s %s failed" % \ ( info.category, info.package, action )) return False EmergeDebug.new_line() return True
while (doUpdateTitle): delta = datetime.datetime.now() - startTime utils.OsUtils.setConsoleTitle("emerge %s %s" % (title, delta)) time.sleep(1) tittleThread = threading.Thread(target=updateTitle, args=( datetime.datetime.now(), " ".join(sys.argv[1:]), )) tittleThread.setDaemon(True) tittleThread.start() success = main() except KeyboardInterrupt: pass except portage.PortageException as e: if e.exception: EmergeDebug.debug(e.exception, 0) traceback.print_tb(e.exception.__traceback__) EmergeDebug.error(e) except Exception as e: print(e) traceback.print_tb(e.__traceback__) finally: doUpdateTitle = False if emergeSettings.getboolean("EmergeDebug", "DumpSettings", False): emergeSettings.dump() if not success: exit(1)
def createPatch(self): """ unpacking all zipped(gz, zip, bz2) tarballs a second time and making a patch """ diffExe = os.path.join(self.rootdir, "dev-utils", "bin", "diff.exe") if not os.path.exists(diffExe): EmergeDebug.die( "could not find diff tool, please run 'emerge diffutils'") # get the file paths of the tarballs filenames = self.localFileNames() destdir = self.workDir() # it makes no sense to make a diff against nothing if (not os.path.exists(self.sourceDir())): EmergeDebug.error( "source directory doesn't exist, please run unpack first") return False EmergeDebug.debug("unpacking files into work root %s" % destdir, 1) # make a temporary directory so the original packages don't overwrite the already existing ones tmpdir = os.path.join(destdir, "tmp") unpackDir = tmpdir utils.cleanDirectory(unpackDir) if (not os.path.exists(unpackDir)): os.mkdir(unpackDir) if (not os.path.exists(unpackDir)): os.mkdir(unpackDir) # unpack all packages for filename in filenames: EmergeDebug.debug("unpacking this file: %s" % filename, 1) if (not utils.unpackFile(EmergeStandardDirs.downloadDir(), filename, unpackDir)): return False packagelist = os.listdir(tmpdir) # apply all patches only ommitting the last one, this makes it possible to always work on the latest patch # for future work, it might be interesting to switch patches on and off at will, this probably needs an # own patch management though patchName = None if self.subinfo.hasTarget() or self.subinfo.hasSvnTarget(): patches = self.subinfo.patchesToApply() if not isinstance(patches, list): patches = list([patches]) for fileName, patchdepth in patches[:-1]: EmergeDebug.debug("applying patch %s with patchlevel: %s" % (fileName, patchdepth)) if not self.applyPatch(fileName, patchdepth, os.path.join(tmpdir, packagelist[0])): return False if patches[-1][0]: patchName = os.path.join(self.buildRoot(), patches[-1][0]) # move the packages up and rename them to be different from the original source directory for directory in packagelist: # if the source or dest directory already exists, remove the occurance instead if os.path.exists(os.path.join(destdir, directory + ".orig")): shutil.rmtree(os.path.join(destdir, directory + ".orig")) shutil.move(os.path.join(tmpdir, directory), os.path.join(destdir, directory + ".orig")) os.chdir(destdir) for directory in packagelist: if not patchName: patchName = os.path.join( self.buildRoot(), "%s-%s.diff" % ( directory, \ str( datetime.date.today() ).replace('-', '') ) ) cmd = "diff -Nrub -x *~ -x *\.rej -x *\.orig -x*\.o %s.orig %s > %s || echo 0" % ( directory, directory, patchName) if not self.system(cmd): return False EmergeDebug.debug("patch created at %s" % patchName) # remove all directories that are not needed any more after making the patch # disabled for now #for directory in packagelist: # shutil.rmtree( directory + ".orig" ) # remove the temporary directory, it should be empty after all directories have been moved up os.rmdir(tmpdir) return True
def createPatch( self ): """ unpacking all zipped(gz, zip, bz2) tarballs a second time and making a patch """ diffExe = os.path.join( self.rootdir, "dev-utils", "bin", "diff.exe" ) if not os.path.exists( diffExe ): EmergeDebug.die("could not find diff tool, please run 'emerge diffutils'") # get the file paths of the tarballs filenames = self.localFileNames() # if using BinaryBuildSystem the files should be unpacked into imagedir if self.buildSystemType == 'binary': destdir = self.installDir() EmergeDebug.debug("unpacking files into image root %s" % destdir, 1) else: destdir = self.workDir() # it makes no sense to make a diff against nothing if ( not os.path.exists( self.sourceDir() ) ): EmergeDebug.error("source directory doesn't exist, please run unpack first") return False EmergeDebug.debug("unpacking files into work root %s" % destdir, 1) # make a temporary directory so the original packages don't overwrite the already existing ones tmpdir = os.path.join( destdir, "tmp" ) unpackDir = tmpdir if ( not os.path.exists( unpackDir ) ): os.mkdir( unpackDir ) if hasattr( self.subinfo.options.unpack, 'unpackDir' ): unpackDir = os.path.join( unpackDir, self.subinfo.options.unpack.unpackDir ) if ( not os.path.exists( unpackDir ) ): os.mkdir( unpackDir ) # unpack all packages for filename in filenames: EmergeDebug.debug("unpacking this file: %s" % filename, 1) if ( not utils.unpackFile( EmergeStandardDirs.downloadDir(), filename, unpackDir ) ): return False packagelist = os.listdir( tmpdir ) # apply all patches only ommitting the last one, this makes it possible to always work on the latest patch # for future work, it might be interesting to switch patches on and off at will, this probably needs an # own patch management though patchName = None if self.subinfo.hasTarget() or self.subinfo.hasSvnTarget(): patches = self.subinfo.patchesToApply() if not isinstance(patches, list): patches = list([patches]) for fileName, patchdepth in patches[:-1]: EmergeDebug.debug("applying patch %s with patchlevel: %s" % (fileName, patchdepth)) if not self.applyPatch( fileName, patchdepth, os.path.join( tmpdir, packagelist[ 0 ] ) ): return False if patches[-1][0]: patchName = os.path.join( self.buildRoot(), patches[-1][0] ) # move the packages up and rename them to be different from the original source directory for directory in packagelist: # if the source or dest directory already exists, remove the occurance instead if os.path.exists( os.path.join( destdir, directory + ".orig" ) ): shutil.rmtree( os.path.join( destdir, directory + ".orig" ) ) shutil.move( os.path.join( tmpdir, directory ), os.path.join( destdir, directory + ".orig" ) ) os.chdir( destdir ) for directory in packagelist: if not patchName: patchName = os.path.join( self.buildRoot(), "%s-%s.diff" % ( directory, \ str( datetime.date.today() ).replace('-', '') ) ) cmd = "diff -Nrub -x *~ -x *\.rej -x *\.orig -x*\.o %s.orig %s > %s || echo 0" % ( directory, directory, patchName ) if not self.system( cmd ): return False EmergeDebug.debug("patch created at %s" % patchName) # remove all directories that are not needed any more after making the patch # disabled for now #for directory in packagelist: # shutil.rmtree( directory + ".orig" ) # remove the temporary directory, it should be empty after all directories have been moved up os.rmdir( tmpdir ) return True
def main( ): parser = argparse.ArgumentParser( prog = "emerge", description = "Emerge is a tool for building KDE-related software under Windows. emerge automates it, looks for the dependencies and fetches them automatically.\ Some options should be used with extreme caution since they will make your kde installation unusable in 999 out of 1000 cases.", epilog = """More information see the README or http://windows.kde.org/. Send feedback to <*****@*****.**>.""" ) parser.add_argument( "-p", "--probe", action = "store_true", help = "probing: emerge will only look which files it has to build according to the list of installed files and according to the dependencies of the package." ) parser.add_argument( "--list-file", action = "store", help = "Build all packages from the csv file provided" ) _def = emergeSettings.get( "General", "EMERGE_OPTIONS", "" ) if _def == "": _def = [] else: _def = _def.split( ";" ) parser.add_argument( "--options", action = "append", default = _def, help = "Set emerge property from string <OPTIONS>. An example for is \"cmake.openIDE=1\" see options.py for more informations." ) parser.add_argument( "-z", "--outDateVCS", action = "store_true", help = "if packages from version control system sources are installed, it marks them as out of date and rebuilds them (tags are not marked as out of date)." ) parser.add_argument( "-sz", "--outDatePackage", action = "store_true", help = "similar to -z, only that it acts only on the last package, and works as normal on the rest." ) parser.add_argument( "-q", "--stayquiet", action = "store_true", dest = "stayQuiet", help = "quiet: there should be no output - The verbose level should be 0" ) parser.add_argument( "-t", "--buildtests", action = "store_true", dest = "buildTests", default = emergeSettings.getboolean( "General", "EMERGE_BUILDTESTS", False ) ) parser.add_argument( "-c", "--continue", action = "store_true", dest = "doContinue" ) parser.add_argument( "--offline", action = "store_true", default = emergeSettings.getboolean( "General", "WorkOffline", False ), help = "do not try to connect to the internet: KDE packages will try to use an existing source tree and other packages would try to use existing packages in the download directory.\ If that doesn't work, the build will fail." ) parser.add_argument( "-f", "--force", action = "store_true", dest = "forced", default = emergeSettings.getboolean( "General", "EMERGE_FORCED", False ) ) parser.add_argument( "--buildtype", choices = [ "Release", "RelWithDebInfo", "MinSizeRel", "Debug" ], dest = "buildType", default = emergeSettings.get( "General", "EMERGE_BUILDTYPE", "RelWithDebInfo" ), help = "This will override the build type set by the environment option EMERGE_BUILDTYPE ." ) parser.add_argument( "-v", "--verbose", action = "count", default = int( emergeSettings.get( "EmergeDebug", "Verbose", "0" ) ), help = " verbose: increases the verbose level of emerge. Default is 1. verbose level 1 contains some notes from emerge, all output of cmake, make and other programs that are used.\ verbose level 2a dds an option VERBOSE=1 to make and emerge is more verbose highest level is verbose level 3." ) parser.add_argument( "--trace", action = "store", default = int( emergeSettings.get( "General", "EMERGE_TRACE", "0" ) ), type = int ) parser.add_argument( "-i", "--ignoreInstalled", action = "store_true", help = "ignore install: using this option will install a package over an existing install. This can be useful if you want to check some new code and your last build isn't that old." ) parser.add_argument( "-ia", "--ignoreAllInstalled", action = "store_true", help = "ignore all install: using this option will install all package over an existing install. This can be useful if you want to check some new code and your last build isn't that old." ) parser.add_argument( "--target", action = "store", help = "This will override the build of the default target. The default Target is marked with a star in the printout of --print-targets" ) parser.add_argument( "--search", action = "store_true", help = "This will search for a package or a description matching or similar to the search term." ) parser.add_argument( "--nocopy", action = "store_true", default = emergeSettings.getboolean( "General", "EMERGE_NOCOPY", False ), help = "this option is deprecated. In older releases emerge would have copied everything from the SVN source tree to a source directory under KDEROOT\\tmp - currently nocopy is applied\ by default if EMERGE_NOCOPY is not set to \"False\". Be aware that setting EMERGE_NOCOPY to \"False\" might slow down the build process, irritate you and increase the disk space roughly\ by the size of SVN source tree." ) parser.add_argument( "--noclean", action = "store_true", default = emergeSettings.getboolean( "General", "EMERGE_NOCLEAN", False ), help = "this option will try to use an existing build directory. Please handle this option with care - it will possibly break if the directory isn't existing." ) parser.add_argument( "--clean", action = "store_false", dest = "noclean", help = "oposite of --noclean" ) parser.add_argument( "--patchlevel", action = "store", default = emergeSettings.get( "General", "EMERGE_PKGPATCHLVL", "" ), help = "This will add a patch level when used together with --package" ) parser.add_argument( "--log-dir", action = "store", default = emergeSettings.get( "General", "EMERGE_LOG_DIR", "" ), help = "This will log the build output to a logfile in LOG_DIR for each package. Logging information is appended to existing logs." ) parser.add_argument( "--dt", action = "store", choices = [ "both", "runtime", "buildtime" ], default = "both", dest = "dependencyType" ) parser.add_argument( "--update-fast", action = "store_true", help = "If the package is installed from svn/git and the revision did not change all steps after fetch are skipped" ) parser.add_argument( "-d", "--dependencydepth", action = "store", type = int, default = -1, help = "By default emerge resolves the whole dependency graph, this option limits the depth of the graph, so a value of 1 would mean only dependencies defined in that package" ) actionHandler = ActionHandler(parser) for x in sorted( [ "fetch", "unpack", "configure", "compile", "make", "install", "install-deps", "qmerge", "manifest", "package", "unmerge", "test", "checkdigest", "dumpdeps", "full-package", "cleanimage", "cleanbuild", "createpatch", "geturls"] ): actionHandler.addAction( x ) actionHandler.addAction( "update", help = "Update a single package" ) # read-only actions actionHandler.addAction( "print-source-version" ) actionHandler.addAction( "print-package-version" ) actionHandler.addAction( "print-targets", help = "This will show a list of available targets for the package" ) actionHandler.addAction( "print-installed", help = "This will show a list of all packages that are installed currently." ) actionHandler.addAction( "print-installable", help = "This will give you a list of packages that can be installed. Currently you don't need to enter the category and package: only the package will be enough." ) actionHandler.addAction( "print-revision", help = "Print the revision of the package and exit" ) actionHandler.addAction( "print-files", help = "Print the files installed by the package and exit" ) actionHandler.addActionWithArg( "search-file", help = "Print packages owning the file" ) # other actions actionHandler.addActionWithArg( "dump-deps-file", dest = "dumpDepsFile", help = "Output the dependencies of this package as a csv file suitable for emerge server." ) actionHandler.addAction( "generate-jenkins-job") parser.add_argument( "packageNames", nargs = argparse.REMAINDER ) args = parser.parse_args( ) action, error = actionHandler.parseFinalAction(args, "all") if not action: EmergeDebug.error("Failed to parse arguments: %s" % error) return False if args.stayQuiet: EmergeDebug.setVerbose(-1) elif args.verbose: EmergeDebug.setVerbose(args.verbose) emergeSettings.set( "General", "WorkOffline", args.offline ) emergeSettings.set( "General", "EMERGE_NOCOPY", args.nocopy ) emergeSettings.set( "General", "EMERGE_NOCLEAN", args.noclean ) emergeSettings.set( "General", "EMERGE_FORCED", args.forced ) emergeSettings.set( "General", "EMERGE_BUILDTESTS", args.buildTests ) emergeSettings.set( "General", "EMERGE_BUILDTYPE", args.buildType ) emergeSettings.set( "PortageVersions", "DefaultTarget", args.target ) emergeSettings.set( "General", "EMERGE_OPTIONS", ";".join( args.options ) ) emergeSettings.set( "General", "EMERGE_LOG_DIR", args.log_dir ) emergeSettings.set( "General", "EMERGE_TRACE", args.trace ) emergeSettings.set( "General", "EMERGE_PKGPATCHLVL", args.patchlevel ) portage.PortageInstance.options = args.options if args.search: for package in args.packageNames: category = "" if not package.find( "/" ) == -1: (category, package) = package.split( "/" ) portageSearch.printSearch( category, package ) return True if action in [ "install-deps", "update", "update-all", "package" ] or args.update_fast: args.ignoreInstalled = True if action in [ "update", "update-all" ]: args.noclean = True EmergeDebug.debug("buildAction: %s" % action) EmergeDebug.debug("doPretend: %s" % args.probe, 1) EmergeDebug.debug("packageName: %s" % args.packageNames) EmergeDebug.debug("buildType: %s" % args.buildType) EmergeDebug.debug("buildTests: %s" % args.buildTests) EmergeDebug.debug("verbose: %d" % EmergeDebug.verbose(), 1) EmergeDebug.debug("trace: %s" % args.trace, 1) EmergeDebug.debug("KDEROOT: %s" % EmergeStandardDirs.emergeRoot(), 1) EmergeDebug.debug_line() if args.print_installed: InstallDB.printInstalled( ) elif args.print_installable: portage.printInstallables( ) elif args.search_file: portage.printPackagesForFileSearch(args.search_file) elif args.list_file: handleSinglePackage( "", action, args ) else: for packageName in args.packageNames: if not handleSinglePackage( packageName, action, args ): return False return True
def handleSinglePackage( packageName, action, args ): deplist = [ ] packageList = [ ] originalPackageList = [ ] categoryList = [ ] targetDict = dict( ) if action == "update-all": installedPackages = portage.PortageInstance.getInstallables( ) if portage.PortageInstance.isCategory( packageName ): EmergeDebug.debug("Updating installed packages from category " + packageName, 1) else: EmergeDebug.debug("Updating all installed packages", 1) packageList = [ ] for mainCategory, mainPackage in installedPackages: if portage.PortageInstance.isCategory( packageName ) and ( mainCategory != packageName ): continue if InstallDB.installdb.isInstalled( mainCategory, mainPackage, args.buildType ) \ and portage.isPackageUpdateable( mainCategory, mainPackage ): categoryList.append( mainCategory ) packageList.append( mainPackage ) EmergeDebug.debug("Will update packages: " + str(packageList), 1) elif args.list_file: listFileObject = open( args.list_file, 'r' ) for line in listFileObject: if line.strip( ).startswith( '#' ): continue try: cat, pac, tar, _ = line.split( ',' ) except: continue categoryList.append( cat ) packageList.append( pac ) originalPackageList.append( pac ) targetDict[ cat + "/" + pac ] = tar elif packageName: packageList, categoryList = portage.getPackagesCategories( packageName ) for entry in packageList: EmergeDebug.debug("Checking dependencies for: %s" % entry, 1) for mainCategory, entry in zip( categoryList, packageList ): deplist = portage.solveDependencies( mainCategory, entry, deplist, args.dependencyType, maxDepth = args.dependencydepth ) # no package found if len( deplist ) == 0: category = "" if not packageName.find( "/" ) == -1: (category, package) = packageName.split( "/" ) portageSearch.printSearch( category, packageName ) return False for item in deplist: item.enabled = args.ignoreAllInstalled if args.ignoreInstalled and item.category in categoryList and item.package in packageList or packageIsOutdated( item.category, item.package ): item.enabled = True if item.category + "/" + item.package in targetDict: item.target = targetDict[ item.category + "/" + item.package ] if args.target in list( portage.PortageInstance.getAllTargets( item.category, item.package ).keys( ) ): # if no target or a wrong one is defined, simply set the default target here item.target = args.target EmergeDebug.debug("dependency: %s" % item, 1) if not deplist: EmergeDebug.debug("<none>", 1) EmergeDebug.debug_line(1) #for item in deplist: # cat = item[ 0 ] # pac = item[ 1 ] # ver = item[ 2 ] # if portage.isInstalled( cat, pac, ver, buildType) and updateAll and not portage.isPackageUpdateable( cat, pac, ver ): # print "remove:", cat, pac, ver # deplist.remove( item ) if action == "install-deps": # the first dependency is the package itself - ignore it # TODO: why are we our own dependency? del deplist[ 0 ] elif action == "update-direct-deps": for item in deplist: item.enabled = True deplist.reverse( ) # package[0] -> category # package[1] -> package # package[2] -> version info = deplist[ -1 ] if not portage.PortageInstance.isVirtualPackage( info.category, info.package ) and \ not action in [ "all", "install-deps" ,"generate-jenkins-job"] and\ not args.list_file or\ action in ["print-targets"]:#not all commands should be executed on the deps if we are a virtual packages # if a buildAction is given, then do not try to build dependencies # and do the action although the package might already be installed. # This is still a bit problematic since packageName might not be a valid # package # for list files, we also want to handle fetching & packaging per package if not handlePackage( info.category, info.package, action, args.doContinue, args.update_fast ): utils.notify( "Emerge %s failed" % action, "%s of %s/%s failed" % ( action, info.category, info.package), action ) return False utils.notify( "Emerge %s finished" % action, "%s of %s/%s finished" % ( action, info.category, info.package), action ) else: if args.dumpDepsFile: dumpDepsFileObject = open( args.dumpDepsFile, 'w+' ) dumpDepsFileObject.write( "# dependency dump of package %s\n" % ( packageName ) ) for info in deplist: isVCSTarget = False if args.dumpDepsFile: dumpDepsFileObject.write( ",".join( [ info.category, info.package, info.target, "" ] ) + "\n" ) isLastPackage = info == deplist[ -1 ] if args.outDateVCS or (args.outDatePackage and isLastPackage): isVCSTarget = portage.PortageInstance.getUpdatableVCSTargets( info.category, info.package ) != [ ] isInstalled = InstallDB.installdb.isInstalled( info.category, info.package ) if args.list_file and action != "all": info.enabled = info.package in originalPackageList if ( isInstalled and not info.enabled ) and not ( isInstalled and (args.outDateVCS or ( args.outDatePackage and isLastPackage) ) and isVCSTarget ): if EmergeDebug.verbose() > 1 and info.package == packageName: EmergeDebug.warning("already installed %s/%s" % (info.category, info.package)) elif EmergeDebug.verbose() > 2 and not info.package == packageName: EmergeDebug.warning("already installed %s/%s" % (info.category, info.package)) else: # in case we only want to see which packages are still to be build, simply return the package name if args.probe: if EmergeDebug.verbose() > 0: EmergeDebug.warning("pretending %s" % info) else: if action in [ "install-deps", "update-direct-deps" ]: action = "all" if not handlePackage( info.category, info.package, action, args.doContinue, args.update_fast ): EmergeDebug.error("fatal error: package %s/%s %s failed" % \ ( info.category, info.package, action )) utils.notify( "Emerge build failed", "Build of %s/%s failed" % ( info.category, info.package), action ) return False utils.notify( "Emerge build finished", "Build of %s/%s finished" % ( info.category, info.package), action ) EmergeDebug.new_line() return True
def updateTitle( startTime, title ): while ( doUpdateTitle ): delta = datetime.datetime.now( ) - startTime utils.setConsoleTitle( "emerge %s %s" % (title, delta) ) time.sleep( 1 ) tittleThread = threading.Thread( target = updateTitle, args = (datetime.datetime.now( ), " ".join( sys.argv[ 1: ] ),) ) tittleThread.setDaemon( True ) tittleThread.start( ) success = main() except KeyboardInterrupt: pass except portage.PortageException as e: if e.exception: EmergeDebug.debug(e.exception, 0) traceback.print_tb( e.exception.__traceback__) EmergeDebug.error(e) except Exception as e: print( e ) traceback.print_tb( e.__traceback__ ) finally: utils.stopTimer( "Emerge" ) doUpdateTitle = False if emergeSettings.getboolean( "EmergeDebug", "DumpSettings", False ): emergeSettings.dump( ) if not success: exit( 1 )