def getPackageInstance(self, category, package):
"""return instance of class Package from package file"""
fileName = getFilename( category, package )
pack = None
mod = None
if fileName.endswith(".py") and os.path.isfile(fileName):
if not fileName in self._packageDict:
EmergeDebug.debug("module to import: %s" % fileName, 2)
if not os.path.isfile( fileName ):
try:
mod = builtins.__import__( fileName )
except ImportError as e:
EmergeDebug.warning('import failed for module %s: %s' % (fileName, str(e)))
mod = None
else:
modulename = os.path.basename( fileName )[:-3].replace('.', '_')
loader = importlib.machinery.SourceFileLoader(modulename, fileName)
try:
mod = loader.load_module()
except Exception as e:
raise PortageException("Failed to load file %s" % fileName, category, package, e)
if not mod is None:
subpackage, package = getSubPackage( category, package )
self._CURRENT_MODULE = ( fileName, category,subpackage, package, mod )
pack = mod.Package( )
self._packageDict[ fileName ] = pack
else:
raise PortageException("Failed to find package", category, package)
else:
pack = self._packageDict[ fileName ]
return pack
def __readDependenciesForChildren(self, deps):
children = []
if deps:
for line in deps:
(category, package) = line.split("/")
EmergeDebug.debug(
"category: %s, name: %s" % (category, package), 2)
try:
version = PortageInstance.getNewestVersion(
category, package)
except PortageException as e:
EmergeDebug.warning(
"%s for %s/%s as a dependency of %s/%s" %
(e, e.category, e.package, self.category, self.name))
continue
if not line in self._dependencyList.keys():
p = DependencyPackage(category, package, False, self)
EmergeDebug.debug(
"adding package %s/%s-%s" %
(category, package, version), 2)
self._dependencyList[line] = p
p.__readChildren()
else:
p = self._dependencyList[line]
children.append(p)
return children
def getNightlyVersionsFromUrl(url, pattern, timeout = 10) -> [str]:
"""
Returns a list of possible version number matching the regular expression in pattern.
:param url: The url to look for the nightly builds.
:param pattern: A regular expression to match the version.
:param timeout:
:return: A list of matching strings or [None]
"""
if emergeSettings.getboolean("General", "WorkOffline"):
EmergeDebug.info("Nightly builds unavailable for %s in offline mode." % url)
return [None]
if url in UtilsCache._NIGTHLY_URLS:
return UtilsCache._NIGTHLY_URLS[url]
else:
try:
with urllib.request.urlopen(url, timeout = timeout) as fh:
data = str(fh.read(), "UTF-8")
vers = re.findall( pattern , data)
if not vers:
print(data)
raise Exception("Pattern %s does not match." % pattern)
UtilsCache._NIGTHLY_URLS[url] = vers
return vers
except Exception as e:
EmergeDebug.warning("Nightly builds unavailable for %s: %s" % (url, e))
return [None]
def doExec(package, action, continueFlag=False):
with EmergeTimer.Timer("%s for %s" % (action, package), 1):
EmergeDebug.info("Action: %s for %s" % (action, package))
ret = package.execute(action)
if not ret:
EmergeDebug.warning("Action: %s for %s FAILED" % (action, package))
return ret or continueFlag
def applyPatch(sourceDir, f, patchLevel='0'):
"""apply single patch"""
cmd = 'patch -d "%s" -p%s -i "%s"' % (sourceDir, patchLevel, f)
EmergeDebug.debug("applying %s" % cmd)
result = system( cmd )
if not result:
EmergeDebug.warning("applying %s failed!" % f)
return result
def rm(path, force=False):
EmergeDebug.debug("deleting file %s" % path, 3)
try:
os.remove(path)
return True
except OSError as e:
EmergeDebug.warning("could not delete file %s: %s" % (path, e))
return False
def rmDir(path, force=False):
EmergeDebug.debug("deleting directory %s" % path, 3)
try:
shutil.rmtree(path)
return True
except OSError as e:
EmergeDebug.warning("could not delete directory %s: %s" % (path, e))
return False
return False
def moveDir( srcdir, destdir ):
""" move directory from srcdir to destdir """
EmergeDebug.debug("moveDir called. srcdir: %s, destdir: %s" % (srcdir, destdir), 1)
try:
shutil.move( srcdir, destdir )
except Exception as e:
EmergeDebug.warning(e)
return False
return True
def system( self, command, errorMessage="", debuglevel=1, **kw):
"""convencience function for running system commands.
This method prints a debug message and then runs a system command.
If the system command returns with errors the method prints an error
message and exits if @ref self.subinfo.options.exitOnErrors is true"""
if utils.system( command, **kw):
return True
if self.subinfo.options.exitOnErrors:
EmergeDebug.warning("while running %s cmd: %s" % (errorMessage, str(command)))
else:
EmergeDebug.warning("while running %s cmd: %s" % (errorMessage, str(command)))
return False
def unmerge( self ):
"""unmergeing the files from the filesystem"""
EmergeDebug.debug("Packagebase unmerge called", 2)
## \todo mergeDestinationDir() reads the real used merge dir from the
## package definition, which fails if this is changed
## a better solution will be to save the merge sub dir into
## /etc/portage/installed and to read from it on unmerge
EmergeDebug.debug("unmerge package from %s" % self.mergeDestinationDir(), 2)
if self.useBuildTypeRelatedMergeRoot and self.subinfo.options.merge.ignoreBuildType \
and self.subinfo.options.merge.destinationPath != None:
for prefix in [ "Release", "RelWithDebInfo", "Debug" ]:
packageList = installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( prefix ) )
for package in packageList:
fileList = package.getFilesWithHashes()
utils.unmergeFileList( self.mergeDestinationDir(), fileList, self.forced )
package.uninstall()
else:
packageList = installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( ) )
for package in packageList:
fileList = package.getFilesWithHashes()
utils.unmergeFileList( self.mergeDestinationDir(), fileList, self.forced )
package.uninstall()
# only packages using a specific merge destination path are shared between build types
if self.useBuildTypeRelatedMergeRoot and self.subinfo.options.merge.ignoreBuildType \
and self.subinfo.options.merge.destinationPath != None:
installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( "Release" ) )
installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( "RelWithDebInfo" ) )
installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( "Debug" ) )
else:
installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( ) )
# run post-uninstall scripts
if not emergeSettings.getboolean("General","EMERGE_NO_POST_INSTALL", False ):
for pkgtype in ['bin', 'lib', 'doc', 'src']:
scriptName = "post-uninstall-%s-%s.cmd" % ( self.package, pkgtype )
script = os.path.join( self.mergeDestinationDir(), "manifest", scriptName )
if os.path.exists( script ):
EmergeDebug.debug("run post uninstall script '%s'" % script, 2)
cmd = "cd /D %s && %s" % ( self.mergeDestinationDir(), script )
if not utils.system(cmd):
EmergeDebug.warning("%s failed!" % cmd)
else:
EmergeDebug.debug("post uninstall script '%s' not found" % script, 2)
else:
EmergeDebug.debug("running of post uninstall scripts disabled!")
return True
def unmerge( self ):
"""unmergeing the files from the filesystem"""
EmergeDebug.debug("Packagebase unmerge called", 2)
## \todo mergeDestinationDir() reads the real used merge dir from the
## package definition, which fails if this is changed
## a better solution will be to save the merge sub dir into
## /etc/portage/installed and to read from it on unmerge
EmergeDebug.debug("unmerge package from %s" % self.mergeDestinationDir(), 2)
if self.useBuildTypeRelatedMergeRoot and self.subinfo.options.merge.ignoreBuildType \
and self.subinfo.options.merge.destinationPath != None:
for prefix in [ "Release", "RelWithDebInfo", "Debug" ]:
packageList = installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( prefix ) )
for package in packageList:
fileList = package.getFilesWithHashes()
utils.unmergeFileList( self.mergeDestinationDir(), fileList, self.forced )
package.uninstall()
else:
packageList = installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( ) )
for package in packageList:
fileList = package.getFilesWithHashes()
utils.unmergeFileList( self.mergeDestinationDir(), fileList, self.forced )
package.uninstall()
# only packages using a specific merge destination path are shared between build types
if self.useBuildTypeRelatedMergeRoot and self.subinfo.options.merge.ignoreBuildType \
and self.subinfo.options.merge.destinationPath != None:
installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( "Release" ) )
installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( "RelWithDebInfo" ) )
installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( "Debug" ) )
else:
installdb.getInstalledPackages( self.category, self.package, self._installedDBPrefix( ) )
# run post-uninstall scripts
if not emergeSettings.getboolean("General","EMERGE_NO_POST_INSTALL", False ):
for pkgtype in ['bin', 'lib', 'doc', 'src']:
scriptName = "post-uninstall-%s-%s.cmd" % ( self.package, pkgtype )
script = os.path.join( self.mergeDestinationDir(), "manifest", scriptName )
if os.path.exists( script ):
EmergeDebug.debug("run post uninstall script '%s'" % script, 2)
cmd = "cd /D %s && %s" % ( self.mergeDestinationDir(), script )
if not utils.system(cmd):
EmergeDebug.warning("%s failed!" % cmd)
else:
EmergeDebug.debug("post uninstall script '%s' not found" % script, 2)
else:
EmergeDebug.debug("running of post uninstall scripts disabled!")
return True
def applyPatch(sourceDir, f, patchLevel='0'):
"""apply single patch"""
cmd = 'patch -d "%s" -p%s < "%s"' % (sourceDir, patchLevel, f)
EmergeDebug.debug("applying %s" % cmd)
if not isCrEol(f):
p = subprocess.Popen([
"patch", "-d", sourceDir, "-p", str(patchLevel)],
stdin = subprocess.PIPE)
p.communicate(bytes(unixToDos(f),'UTF-8'))
result = p.wait() == 0
else:
result = system( cmd )
if not result:
EmergeDebug.warning("applying %s failed!" % f)
return result
def system(self, command, errorMessage="", debuglevel=1, **kw):
"""convencience function for running system commands.
This method prints a debug message and then runs a system command.
If the system command returns with errors the method prints an error
message and exits if @ref self.subinfo.options.exitOnErrors is true"""
if utils.system(command, **kw):
return True
if self.subinfo.options.exitOnErrors:
EmergeDebug.warning("while running %s cmd: %s" %
(errorMessage, str(command)))
else:
EmergeDebug.warning("while running %s cmd: %s" %
(errorMessage, str(command)))
return False
def install( self):
if not CMakeBuildSystem.install(self):
return False
for root, _, svgs in os.walk(self.imageDir(), ):
for svg in svgs:
path = os.path.join( root, svg)
if path.endswith(".svg") and os.path.isfile(path):
toReplace = self.resolveGitSymLink(path)
if not os.path.exists(toReplace):
EmergeDebug.warning("Resolving %s failed: %s does not exists." % (path, toReplace))
continue
if toReplace != path:
utils.deleteFile(path)
utils.copyFile( toReplace, path)
return True
def qmerge( self ):
"""mergeing the imagedirectory into the filesystem"""
## \todo is this the optimal place for creating the post install scripts ?
ignoreInstalled = False
prefixPath = self._installedDBPrefix( self.buildType() )
if installdb.isInstalled( category=None, package=self.package, prefix=prefixPath ):
ignoreInstalled = True
self.unmerge()
EmergeDebug.debug("qmerge package to %s" % self.mergeDestinationDir(), 2)
utils.mergeImageDirToRootDir( self.mergeSourceDir(), self.mergeDestinationDir() )
# run post-install scripts
if not emergeSettings.getboolean("General","EMERGE_NO_POST_INSTALL", False ):
for pkgtype in ['bin', 'lib', 'doc', 'src']:
scriptName = "post-install-%s-%s.cmd" % ( self.package, pkgtype )
script = os.path.join( self.mergeDestinationDir(), "manifest", scriptName )
if os.path.exists( script ):
EmergeDebug.debug("run post install script '%s'" % script, 2)
cmd = "cd /D %s && %s" % ( self.mergeDestinationDir(), script )
if not utils.system(cmd):
EmergeDebug.warning("%s failed!" % cmd)
else:
EmergeDebug.debug("post install script '%s' not found" % script, 2)
else:
EmergeDebug.debug("running of post install scripts disabled!")
# add package to installed database -> is this not the task of the manifest files ?
# only packages using a specific merge destination path are shared between build types
revision = self.sourceRevision()
if self.useBuildTypeRelatedMergeRoot and self.subinfo.options.merge.ignoreBuildType \
and self.subinfo.options.merge.destinationPath != None:
for prefix in [ "Release", "RelWithDebInfo", "Debug" ]:
package = installdb.addInstalled( self.category, self.package, self.version, self._installedDBPrefix( prefix ), ignoreInstalled, revision = revision)
package.addFiles( utils.getFileListFromDirectory( self._installedDBPrefix( prefix ) ) )
package.install()
else:
package = installdb.addInstalled( self.category, self.package, self.version, self._installedDBPrefix(), ignoreInstalled, revision = revision )
package.addFiles( utils.getFileListFromDirectory( self.mergeSourceDir() ) )
package.install()
return True
def qmerge( self ):
"""mergeing the imagedirectory into the filesystem"""
## \todo is this the optimal place for creating the post install scripts ?
ignoreInstalled = False
prefixPath = self._installedDBPrefix( self.buildType() )
if installdb.isInstalled( category=None, package=self.package, prefix=prefixPath ):
ignoreInstalled = True
self.unmerge()
EmergeDebug.debug("qmerge package to %s" % self.mergeDestinationDir(), 2)
utils.mergeImageDirToRootDir( self.mergeSourceDir(), self.mergeDestinationDir() )
# run post-install scripts
if not emergeSettings.getboolean("General","EMERGE_NO_POST_INSTALL", False ):
for pkgtype in ['bin', 'lib', 'doc', 'src']:
scriptName = "post-install-%s-%s.cmd" % ( self.package, pkgtype )
script = os.path.join( self.mergeDestinationDir(), "manifest", scriptName )
if os.path.exists( script ):
EmergeDebug.debug("run post install script '%s'" % script, 2)
cmd = "cd /D %s && %s" % ( self.mergeDestinationDir(), script )
if not utils.system(cmd):
EmergeDebug.warning("%s failed!" % cmd)
else:
EmergeDebug.debug("post install script '%s' not found" % script, 2)
else:
EmergeDebug.debug("running of post install scripts disabled!")
# add package to installed database -> is this not the task of the manifest files ?
# only packages using a specific merge destination path are shared between build types
revision = self.sourceRevision()
if self.useBuildTypeRelatedMergeRoot and self.subinfo.options.merge.ignoreBuildType \
and self.subinfo.options.merge.destinationPath != None:
for prefix in [ "Release", "RelWithDebInfo", "Debug" ]:
package = installdb.addInstalled( self.category, self.package, self.version, self._installedDBPrefix( prefix ), ignoreInstalled, revision = revision)
package.addFiles( utils.getFileListFromDirectory( self._installedDBPrefix( prefix ) ) )
package.install()
else:
package = installdb.addInstalled( self.category, self.package, self.version, self._installedDBPrefix(), ignoreInstalled, revision = revision )
package.addFiles( utils.getFileListFromDirectory( self.mergeSourceDir() ) )
package.install()
return True
def copyFile(src, dest,linkOnly = emergeSettings.getboolean("General", "UseHardlinks", False)):
""" copy file from src to dest"""
EmergeDebug.debug("copy file from %s to %s" % (src, dest), 2)
destDir = os.path.dirname( dest )
if not os.path.exists( destDir ):
os.makedirs( destDir )
if os.path.exists( dest ):
EmergeDebug.warning("Overriding %s" % dest)
OsUtils.rm( dest, True )
if linkOnly:
try:
os.link( src , dest )
return True
except:
EmergeDebug.warning("Failed to create hardlink %s for %s" % (dest, src))
shutil.copy(src,dest)
return True
def install(self):
if not CMakeBuildSystem.install(self):
return False
for root, _, svgs in os.walk(self.imageDir(), ):
for svg in svgs:
path = os.path.join(root, svg)
if path.endswith(".svg") and os.path.isfile(path):
toReplace = self.resolveGitSymLink(path)
if not os.path.exists(toReplace):
EmergeDebug.warning(
"Resolving %s failed: %s does not exists." %
(path, toReplace))
continue
if toReplace != path:
utils.deleteFile(path)
utils.copyFile(toReplace, path)
return True
def __readDependenciesForChildren( self, deps):
children = []
if deps:
for line in deps:
( category, package ) = line.split( "/" )
EmergeDebug.debug("category: %s, name: %s" % (category, package), 2)
try:
version = PortageInstance.getNewestVersion( category, package )
except PortageException as e:
EmergeDebug.warning("%s for %s/%s as a dependency of %s/%s" % (e, e.category, e.package, self.category , self.name))
continue
if not line in self._dependencyList.keys():
p = DependencyPackage( category, package, False, self )
EmergeDebug.debug("adding package %s/%s-%s" % (category, package, version), 2)
self._dependencyList[ line ] = p
p.__readChildren()
else:
p = self._dependencyList[ line ]
children.append( p )
return children
def unmergeFileList(rootdir, fileList, forced=False):
""" delete files in the fileList if has matches or forced is True """
for filename, filehash in fileList:
fullPath = os.path.join(rootdir, os.path.normcase( filename))
if os.path.isfile(fullPath):
algorithm = EmergeHash.HashAlgorithm.getAlgorithmFromPrefix(filehash)
if not algorithm:
currentHash = EmergeHash.digestFile(fullPath, EmergeHash.HashAlgorithm.MD5)
else:
currentHash = algorithm.stringPrefix() + EmergeHash.digestFile(fullPath, algorithm)
if currentHash == filehash or filehash == "":
EmergeDebug.debug("deleting file %s" % fullPath, 2)
try:
os.remove(fullPath)
except OSError:
system( "cmd /C \"attrib -R %s\"" % fullPath )
os.remove(fullPath)
else:
if forced:
EmergeDebug.warning("file %s has different hash: %s %s, deleting anyway" % \
(fullPath, currentHash, filehash ))
try:
os.remove(fullPath)
except OSError:
system( "cmd /C \"attrib -R %s\"" % fullPath )
os.remove(fullPath)
else:
EmergeDebug.warning("file %s has different hash: %s %s, run with option --force to delete it anyway" % \
(fullPath, currentHash, filehash ))
elif not os.path.isdir(fullPath):
EmergeDebug.warning("file %s does not exist" % fullPath)
def unpackFile( downloaddir, filename, workdir ):
"""unpack file specified by 'filename' from 'downloaddir' into 'workdir'"""
( shortname, ext ) = os.path.splitext( filename )
if ( ext == ".zip" ):
return unZip( os.path.join( downloaddir, filename ), workdir )
elif ( ext == ".7z" ):
return un7zip( os.path.join( downloaddir, filename ), workdir, ext )
elif ( ext == ".tgz" ):
return unTar( os.path.join( downloaddir, filename ), workdir )
elif ( ext == ".gz" or ext == ".bz2" or ext == ".lzma" or ext == ".xz" ):
_, myext = os.path.splitext( shortname )
if ( myext == ".tar" ):
return unTar( os.path.join( downloaddir, filename ), workdir )
else:
EmergeDebug.error("unpacking %s" % myext)
return False
elif ( ext == ".exe" ):
EmergeDebug.warning("unpack ignoring exe file")
return True
else:
EmergeDebug.error("dont know how to unpack this file: %s" % filename)
return False
def createImportLibs( dll_name, basepath ):
"""creating the import libraries for the other compiler(if ANSI-C libs)"""
dst = os.path.join( basepath, "lib" )
if( not os.path.exists( dst ) ):
os.mkdir( dst )
# check whether the required binary tools exist
HAVE_GENDEF = UtilsCache.findApplication( "gendef" ) is not None
USE_GENDEF = HAVE_GENDEF
HAVE_LIB = UtilsCache.findApplication( "lib" ) is not None
HAVE_DLLTOOL = UtilsCache.findApplication( "dlltool" ) is not None
if EmergeDebug.verbose() > 1:
print("gendef found:", HAVE_GENDEF)
print("gendef used:", USE_GENDEF)
print("lib found:", HAVE_LIB)
print("dlltool found:", HAVE_DLLTOOL)
dllpath = os.path.join( basepath, "bin", "%s.dll" % dll_name )
defpath = os.path.join( basepath, "lib", "%s.def" % dll_name )
exppath = os.path.join( basepath, "lib", "%s.exp" % dll_name )
imppath = os.path.join( basepath, "lib", "%s.lib" % dll_name )
gccpath = os.path.join( basepath, "lib", "%s.dll.a" % dll_name )
if not HAVE_GENDEF and os.path.exists( defpath ):
HAVE_GENDEF = True
USE_GENDEF = False
if not HAVE_GENDEF:
EmergeDebug.warning("system does not have gendef.exe")
return False
if not HAVE_LIB and not os.path.isfile( imppath ):
EmergeDebug.warning("system does not have lib.exe (from msvc)")
if not HAVE_DLLTOOL and not os.path.isfile( gccpath ):
EmergeDebug.warning("system does not have dlltool.exe")
# create .def
if USE_GENDEF:
cmd = "gendef - %s -a > %s " % ( dllpath, defpath )
system( cmd )
if( HAVE_LIB and not os.path.isfile( imppath ) ):
# create .lib
cmd = "lib /machine:x86 /def:%s /out:%s" % ( defpath, imppath )
system( cmd )
if( HAVE_DLLTOOL and not os.path.isfile( gccpath ) ):
# create .dll.a
cmd = "dlltool -d %s -l %s -k" % ( defpath, gccpath )
system( cmd )
if os.path.exists( defpath ):
os.remove( defpath )
if os.path.exists( exppath ):
os.remove( exppath )
return True
def unTar( fileName, destdir ):
"""unpack tar file specified by 'file' into 'destdir'"""
EmergeDebug.debug("unTar called. file: %s, destdir: %s" % (fileName, destdir), 1)
( shortname, ext ) = os.path.splitext( fileName )
emerge_tmp = os.path.join(destdir,"emerge_tmp")
mode = "r"
if ( ext == ".gz" ):
mode = "r:gz"
#elif(ext == ".bz2"):
#mode = "r:bz2"
elif(ext == ".lzma" or ext == ".xz" or ext == ".bz2"):
un7zip( fileName, emerge_tmp )
_, tarname = os.path.split( shortname )
fileName = os.path.join( emerge_tmp , tarname )
if not os.path.exists( fileName ):
EmergeDebug.error("couldn't find file %s" % fileName)
return False
try:
with tarfile.open( fileName, mode ) as tar:
# FIXME how to handle errors here ?
for tarMember in tar:
try:
if tarMember.issym():
tarDir = os.path.dirname(tarMember.name)
target = tarMember.linkname
if not target.startswith("/"):#abspath?
target = os.path.normpath("%s/%s"%(tarDir, target)).replace("\\","/")
if target in tar.getnames():
tar.extract(target, emerge_tmp )
shutil.move(os.path.join( emerge_tmp , tarDir , tarMember.linkname ),os.path.join( destdir , tarMember.name ))
EmergeDebug.warning("Resolved symlink %s in tarfile %s to %s" % (tarMember.name, fileName , tarMember.linkname))
else:
EmergeDebug.warning("link target %s for %s not included in tarfile" % (target , tarMember.name))
else:
tar.extract(tarMember, destdir )
except tarfile.TarError:
EmergeDebug.error("couldn't extract file %s to directory %s" % (fileName, destdir))
return False
except IOError:
EmergeDebug.warning("Failed to extract %s to directory %s" % (tarMember.name, destdir))
return True
except tarfile.TarError as e:
EmergeDebug.error("could not open existing tar archive: %s error: %s" % (fileName, e))
return False
finally:
if os.path.exists(emerge_tmp):
shutil.rmtree(emerge_tmp)
def unmergeFileList(rootdir, fileList, forced=False):
""" delete files in the fileList if has matches or forced is True """
for filename, filehash in fileList:
fullPath = os.path.join(rootdir, os.path.normcase( filename))
if os.path.isfile(fullPath):
algorithm = EmergeHash.HashAlgorithm.getAlgorithmFromPrefix(filehash)
if not algorithm:
currentHash = EmergeHash.digestFile(fullPath, EmergeHash.HashAlgorithm.MD5)
else:
currentHash = algorithm.stringPrefix() + EmergeHash.digestFile(fullPath, algorithm)
if currentHash == filehash or filehash == "":
OsUtils.rm(fullPath, True)
else:
if forced:
EmergeDebug.warning("file %s has different hash: %s %s, deleting anyway" % \
(fullPath, currentHash, filehash ))
OsUtils.rm(fullPath, True)
else:
EmergeDebug.warning("file %s has different hash: %s %s, run with option --force to delete it anyway" % \
(fullPath, currentHash, filehash ))
elif not os.path.isdir(fullPath):
EmergeDebug.warning("file %s does not exist" % fullPath)
def createPackage(self):
"""packaging according to the gnuwin32 packaging rules.
This requires the kdewin-packager"""
if not self.packagerExe:
EmergeDebug.die("could not find kdewin-packager in your path!")
if self.subinfo.options.package.packageName != None:
pkgName = self.subinfo.options.package.packageName
else:
pkgName = self.package
if pkgName.endswith('-src') or pkgName.endswith('-pkg'):
pkgName = pkgName[:-4]
pkgVersion, pkgNotesVersion = self.getPackageVersion()
# kdewin packager creates his own manifest files, so there is no need to add
# if self.subinfo.options.package.withDigests:
# utils.createManifestFiles(filesDir, filesDir, "", self.package, pkgVersion)
# FIXME: add a test for the installer later
dstpath = self.packageDestinationDir()
if ( self.subinfo.options.package.packSources ) and os.path.exists( self.sourceDir() ):
srcCmd = " -srcroot " + self.sourceDir()
else:
if not os.path.exists( self.sourceDir() ):
EmergeDebug.warning("The source directory %s doesn't exist and can't be used for packaging." % self.sourceDir())
srcCmd = ""
# copy pdb/sym files to a temporary directory, because they can be scattered all over the build directory
# plus, different build types copy files to different directories (could be buildDir(), buildDir()/bin, buildDir()/bin/Debug...),
# so let's copy them to one precise location, package them, then delete that directory
symCmd = ""
if self.useDebugPackages:
# directories to ignore: cmake temporary files, and dbg so it won't try to copy a file to itself
dirsToIgnore = [ 'cmake', 'CMakeFiles', 'CMakeTmp', 'CMakeTmp2', 'CMakeTmp3', 'dbg' ]
path = self.buildDir()
# where to copy the debugging information files
symRoot = os.path.join( self.buildDir(), "dbg" )
symPath = os.path.join( symRoot, "bin" )
if not os.path.exists( symPath ):
utils.createDir( symPath )
# shouldn't be needed, usually; but if files are present, that could lead to errors
utils.cleanDirectory ( symRoot )
EmergeDebug.debug("Copying debugging files to 'dbg'...")
for path, _, files in os.walk( path ):
found = 0
for directory in range( 0, len( dirsToIgnore ) ):
if path.find( dirsToIgnore[directory] ) > 0:
found = 1
break
if found == 1:
continue
EmergeDebug.debug("Checking: %s" % path, 3)
for fileName in files:
if ( fileName.endswith( ".pdb" ) ):
utils.copyFile( os.path.join( path, fileName ), os.path.join( symPath, fileName ) )
elif not self.subinfo.options.package.disableStriping:
if ( fileName.endswith( ".exe" ) or fileName.endswith( ".dll" ) or fileName.endswith( ".obf" ) ):
if compiler.isMinGW():
symFilename = fileName[:-4] + ".sym"
utils.system( "strip --only-keep-debug " + " -o " + os.path.join( path, symFilename ) \
+ " " + os.path.join( path, fileName ) )
# utils.system( "strip --strip-all " + os.path.join( path, fileName ) )
utils.copyFile( os.path.join(path, symFilename), os.path.join( symPath, symFilename ) )
if not self.subinfo.options.package.disableStriping and compiler.isMinGW() :
symCmd += "-strip "
symCmd += "-debug-package "
symCmd += "-symroot " + symRoot
EmergeDebug.debug(symCmd, 2)
cmd = "-name %s -root %s -version %s -destdir %s %s %s -checksum sha1 " % \
( pkgName, self.imageDir(), pkgVersion, dstpath, srcCmd, symCmd )
xmltemplate = self.xmlTemplate()
if os.path.exists(xmltemplate):
cmd = self.packagerExe + " " + cmd + " -template " + xmltemplate + " -notes " + \
"%s/%s:%s:unknown " % ( self.category, self.package, pkgNotesVersion ) + "-compression 2 "
EmergeDebug.debug("using xml template for package generating", 1)
elif self.package == "qt":
cmd = self.packagerExe + " " + cmd + " -template :/template-qt.xml -notes " + \
"%s/%s:%s:unknown " % ( self.category, self.package, pkgNotesVersion ) + "-compression 2 "
EmergeDebug.debug("using xml template for package generating", 1)
else:
cmd = self.packagerExe + " " + cmd + " -verbose -notes " + \
"%s/%s:%s:unknown " % ( self.category, self.package, pkgNotesVersion ) + "-compression 2 "
EmergeDebug.debug(" xml template %s for package generating not found" % xmltemplate, 1)
if( self.subinfo.options.package.withCompiler ):
cmd += " -type "
if compiler.isMinGW():
if self.buildArchitecture() == "x64":
cmd += "x64-"
else:
cmd += "x86-"
cmd += compiler.getShortName()
# not needed anymore
# if self.subinfo.options.package.specialMode:
# cmd += " -special"
if not utils.system(cmd):
EmergeDebug.die("while packaging. cmd: %s" % cmd)
if self.useDebugPackages:
utils.rmtree( symPath )
return True
def createPackage(self):
"""packaging according to the gnuwin32 packaging rules.
This requires the kdewin-packager"""
if not self.packagerExe:
EmergeDebug.die("could not find kdewin-packager in your path!")
if self.subinfo.options.package.packageName != None:
pkgName = self.subinfo.options.package.packageName
else:
pkgName = self.package
if pkgName.endswith('-src') or pkgName.endswith('-pkg'):
pkgName = pkgName[:-4]
pkgVersion, pkgNotesVersion = self.getPackageVersion()
# kdewin packager creates his own manifest files, so there is no need to add
# if self.subinfo.options.package.withDigests:
# utils.createManifestFiles(filesDir, filesDir, "", self.package, pkgVersion)
# FIXME: add a test for the installer later
dstpath = self.packageDestinationDir()
if (self.subinfo.options.package.packSources) and os.path.exists(
self.sourceDir()):
srcCmd = " -srcroot " + self.sourceDir()
else:
if not os.path.exists(self.sourceDir()):
EmergeDebug.warning(
"The source directory %s doesn't exist and can't be used for packaging."
% self.sourceDir())
srcCmd = ""
# copy pdb/sym files to a temporary directory, because they can be scattered all over the build directory
# plus, different build types copy files to different directories (could be buildDir(), buildDir()/bin, buildDir()/bin/Debug...),
# so let's copy them to one precise location, package them, then delete that directory
symCmd = ""
if self.useDebugPackages:
# directories to ignore: cmake temporary files, and dbg so it won't try to copy a file to itself
dirsToIgnore = [
'cmake', 'CMakeFiles', 'CMakeTmp', 'CMakeTmp2', 'CMakeTmp3',
'dbg'
]
path = self.buildDir()
# where to copy the debugging information files
symRoot = os.path.join(self.buildDir(), "dbg")
symPath = os.path.join(symRoot, "bin")
if not os.path.exists(symPath):
utils.createDir(symPath)
# shouldn't be needed, usually; but if files are present, that could lead to errors
utils.cleanDirectory(symRoot)
EmergeDebug.debug("Copying debugging files to 'dbg'...")
for path, _, files in os.walk(path):
found = 0
for directory in range(0, len(dirsToIgnore)):
if path.find(dirsToIgnore[directory]) > 0:
found = 1
break
if found == 1:
continue
EmergeDebug.debug("Checking: %s" % path, 3)
for fileName in files:
if (fileName.endswith(".pdb")):
utils.copyFile(os.path.join(path, fileName),
os.path.join(symPath, fileName))
elif not self.subinfo.options.package.disableStriping:
if (fileName.endswith(".exe")
or fileName.endswith(".dll")
or fileName.endswith(".obf")):
if compiler.isMinGW():
symFilename = fileName[:-4] + ".sym"
utils.system( "strip --only-keep-debug " + " -o " + os.path.join( path, symFilename ) \
+ " " + os.path.join( path, fileName ) )
# utils.system( "strip --strip-all " + os.path.join( path, fileName ) )
utils.copyFile(
os.path.join(path, symFilename),
os.path.join(symPath, symFilename))
if not self.subinfo.options.package.disableStriping and compiler.isMinGW(
):
symCmd += "-strip "
symCmd += "-debug-package "
symCmd += "-symroot " + symRoot
EmergeDebug.debug(symCmd, 2)
cmd = "-name %s -root %s -version %s -destdir %s %s %s -checksum sha1 " % \
( pkgName, self.imageDir(), pkgVersion, dstpath, srcCmd, symCmd )
xmltemplate = self.xmlTemplate()
if os.path.exists(xmltemplate):
cmd = self.packagerExe + " " + cmd + " -template " + xmltemplate + " -notes " + \
"%s/%s:%s:unknown " % ( self.category, self.package, pkgNotesVersion ) + "-compression 2 "
EmergeDebug.debug("using xml template for package generating", 1)
elif self.package == "qt":
cmd = self.packagerExe + " " + cmd + " -template :/template-qt.xml -notes " + \
"%s/%s:%s:unknown " % ( self.category, self.package, pkgNotesVersion ) + "-compression 2 "
EmergeDebug.debug("using xml template for package generating", 1)
else:
cmd = self.packagerExe + " " + cmd + " -verbose -notes " + \
"%s/%s:%s:unknown " % ( self.category, self.package, pkgNotesVersion ) + "-compression 2 "
EmergeDebug.debug(
" xml template %s for package generating not found" %
xmltemplate, 1)
if (self.subinfo.options.package.withCompiler):
cmd += " -type "
if compiler.isMinGW():
if self.buildArchitecture() == "x64":
cmd += "x64-"
else:
cmd += "x86-"
cmd += compiler.getShortName()
# not needed anymore
# if self.subinfo.options.package.specialMode:
# cmd += " -special"
if not utils.system(cmd):
EmergeDebug.die("while packaging. cmd: %s" % cmd)
if self.useDebugPackages:
utils.rmtree(symPath)
return True
def findApplication(app) -> str:
appLocation = shutil.which(app)
if not appLocation:
EmergeDebug.warning("Emerge was unable to locate: %s" % app)
return None
return appLocation
def enterSourceDir(self):
if ( not os.path.exists( self.sourceDir() ) ):
return False
EmergeDebug.warning("entering the source directory!")
os.chdir( self.sourceDir() )
EmergeDebug.debug("entering: %s" % self.sourceDir())
def handleSinglePackage(packageName, action, args):
deplist = []
packageList = []
originalPackageList = []
categoryList = []
targetDict = dict()
if action == "update-all":
installedPackages = portage.PortageInstance.getInstallables()
if portage.PortageInstance.isCategory(packageName):
EmergeDebug.debug(
"Updating installed packages from category " + packageName, 1)
else:
EmergeDebug.debug("Updating all installed packages", 1)
packageList = []
for mainCategory, mainPackage in installedPackages:
if portage.PortageInstance.isCategory(packageName) and (
mainCategory != packageName):
continue
if InstallDB.installdb.isInstalled( mainCategory, mainPackage, args.buildType ) \
and portage.isPackageUpdateable( mainCategory, mainPackage ):
categoryList.append(mainCategory)
packageList.append(mainPackage)
EmergeDebug.debug("Will update packages: " + str(packageList), 1)
elif args.list_file:
listFileObject = open(args.list_file, 'r')
for line in listFileObject:
if line.strip().startswith('#'): continue
try:
cat, pac, tar, _ = line.split(',')
except:
continue
categoryList.append(cat)
packageList.append(pac)
originalPackageList.append(pac)
targetDict[cat + "/" + pac] = tar
elif packageName:
packageList, categoryList = portage.getPackagesCategories(packageName)
for entry in packageList:
EmergeDebug.debug("Checking dependencies for: %s" % entry, 1)
for mainCategory, entry in zip(categoryList, packageList):
deplist = portage.solveDependencies(mainCategory,
entry,
deplist,
args.dependencyType,
maxDepth=args.dependencydepth)
# no package found
if len(deplist) == 0:
category = ""
if not packageName.find("/") == -1:
(category, package) = packageName.split("/")
portageSearch.printSearch(category, packageName)
return False
for item in deplist:
item.enabled = args.ignoreAllInstalled
if args.ignoreInstalled and item.category in categoryList and item.package in packageList or packageIsOutdated(
item.category, item.package):
item.enabled = True
if item.category + "/" + item.package in targetDict:
item.target = targetDict[item.category + "/" + item.package]
if args.target in list(
portage.PortageInstance.getAllTargets(item.category,
item.package).keys()):
# if no target or a wrong one is defined, simply set the default target here
item.target = args.target
EmergeDebug.debug("dependency: %s" % item, 1)
if not deplist:
EmergeDebug.debug("<none>", 1)
EmergeDebug.debug_line(1)
#for item in deplist:
# cat = item[ 0 ]
# pac = item[ 1 ]
# ver = item[ 2 ]
# if portage.isInstalled( cat, pac, ver, buildType) and updateAll and not portage.isPackageUpdateable( cat, pac, ver ):
# print "remove:", cat, pac, ver
# deplist.remove( item )
if action == "install-deps":
# the first dependency is the package itself - ignore it
# TODO: why are we our own dependency?
del deplist[0]
elif action == "update-direct-deps":
for item in deplist:
item.enabled = True
deplist.reverse()
# package[0] -> category
# package[1] -> package
# package[2] -> version
info = deplist[-1]
if not portage.PortageInstance.isVirtualPackage( info.category, info.package ) and \
not action in [ "all", "install-deps"] and\
not args.list_file or\
action in ["print-targets"]:#not all commands should be executed on the deps if we are a virtual packages
# if a buildAction is given, then do not try to build dependencies
# and do the action although the package might already be installed.
# This is still a bit problematic since packageName might not be a valid
# package
# for list files, we also want to handle fetching & packaging per package
if not handlePackage(info.category, info.package, action,
args.doContinue, args.update_fast):
return False
else:
if args.dumpDepsFile:
dumpDepsFileObject = open(args.dumpDepsFile, 'w+')
dumpDepsFileObject.write("# dependency dump of package %s\n" %
(packageName))
for info in deplist:
isVCSTarget = False
if args.dumpDepsFile:
dumpDepsFileObject.write(
",".join([info.category, info.package, info.target, ""]) +
"\n")
isLastPackage = info == deplist[-1]
if args.outDateVCS or (args.outDatePackage and isLastPackage):
isVCSTarget = portage.PortageInstance.getUpdatableVCSTargets(
info.category, info.package) != []
isInstalled = InstallDB.installdb.isInstalled(
info.category, info.package)
if args.list_file and action != "all":
info.enabled = info.package in originalPackageList
if (isInstalled and not info.enabled) and not (
isInstalled and
(args.outDateVCS or
(args.outDatePackage and isLastPackage)) and isVCSTarget):
if EmergeDebug.verbose() > 1 and info.package == packageName:
EmergeDebug.warning("already installed %s/%s" %
(info.category, info.package))
elif EmergeDebug.verbose(
) > 2 and not info.package == packageName:
EmergeDebug.warning("already installed %s/%s" %
(info.category, info.package))
else:
# in case we only want to see which packages are still to be build, simply return the package name
if args.probe:
EmergeDebug.warning("pretending %s" % info)
else:
if action in ["install-deps", "update-direct-deps"]:
action = "all"
if not handlePackage(info.category, info.package, action,
args.doContinue, args.update_fast):
EmergeDebug.error("fatal error: package %s/%s %s failed" % \
( info.category, info.package, action ))
return False
EmergeDebug.new_line()
return True
def handleSinglePackage( packageName, action, args ):
deplist = [ ]
packageList = [ ]
originalPackageList = [ ]
categoryList = [ ]
targetDict = dict( )
if action == "update-all":
installedPackages = portage.PortageInstance.getInstallables( )
if portage.PortageInstance.isCategory( packageName ):
EmergeDebug.debug("Updating installed packages from category " + packageName, 1)
else:
EmergeDebug.debug("Updating all installed packages", 1)
packageList = [ ]
for mainCategory, mainPackage in installedPackages:
if portage.PortageInstance.isCategory( packageName ) and ( mainCategory != packageName ):
continue
if InstallDB.installdb.isInstalled( mainCategory, mainPackage, args.buildType ) \
and portage.isPackageUpdateable( mainCategory, mainPackage ):
categoryList.append( mainCategory )
packageList.append( mainPackage )
EmergeDebug.debug("Will update packages: " + str(packageList), 1)
elif args.list_file:
listFileObject = open( args.list_file, 'r' )
for line in listFileObject:
if line.strip( ).startswith( '#' ): continue
try:
cat, pac, tar, _ = line.split( ',' )
except:
continue
categoryList.append( cat )
packageList.append( pac )
originalPackageList.append( pac )
targetDict[ cat + "/" + pac ] = tar
elif packageName:
packageList, categoryList = portage.getPackagesCategories( packageName )
for entry in packageList:
EmergeDebug.debug("Checking dependencies for: %s" % entry, 1)
for mainCategory, entry in zip( categoryList, packageList ):
deplist = portage.solveDependencies( mainCategory, entry, deplist, args.dependencyType,
maxDepth = args.dependencydepth )
# no package found
if len( deplist ) == 0:
category = ""
if not packageName.find( "/" ) == -1:
(category, package) = packageName.split( "/" )
portageSearch.printSearch( category, packageName )
return False
for item in deplist:
item.enabled = args.ignoreAllInstalled
if args.ignoreInstalled and item.category in categoryList and item.package in packageList or packageIsOutdated(
item.category, item.package ):
item.enabled = True
if item.category + "/" + item.package in targetDict:
item.target = targetDict[ item.category + "/" + item.package ]
if args.target in list(
portage.PortageInstance.getAllTargets( item.category, item.package ).keys( ) ):
# if no target or a wrong one is defined, simply set the default target here
item.target = args.target
EmergeDebug.debug("dependency: %s" % item, 1)
if not deplist:
EmergeDebug.debug("<none>", 1)
EmergeDebug.debug_line(1)
#for item in deplist:
# cat = item[ 0 ]
# pac = item[ 1 ]
# ver = item[ 2 ]
# if portage.isInstalled( cat, pac, ver, buildType) and updateAll and not portage.isPackageUpdateable( cat, pac, ver ):
# print "remove:", cat, pac, ver
# deplist.remove( item )
if action == "install-deps":
# the first dependency is the package itself - ignore it
# TODO: why are we our own dependency?
del deplist[ 0 ]
elif action == "update-direct-deps":
for item in deplist:
item.enabled = True
deplist.reverse( )
# package[0] -> category
# package[1] -> package
# package[2] -> version
info = deplist[ -1 ]
if not portage.PortageInstance.isVirtualPackage( info.category, info.package ) and \
not action in [ "all", "install-deps" ,"generate-jenkins-job"] and\
not args.list_file or\
action in ["print-targets"]:#not all commands should be executed on the deps if we are a virtual packages
# if a buildAction is given, then do not try to build dependencies
# and do the action although the package might already be installed.
# This is still a bit problematic since packageName might not be a valid
# package
# for list files, we also want to handle fetching & packaging per package
if not handlePackage( info.category, info.package, action, args.doContinue, args.update_fast ):
utils.notify( "Emerge %s failed" % action, "%s of %s/%s failed" % (
action, info.category, info.package), action )
return False
utils.notify( "Emerge %s finished" % action,
"%s of %s/%s finished" % ( action, info.category, info.package),
action )
else:
if args.dumpDepsFile:
dumpDepsFileObject = open( args.dumpDepsFile, 'w+' )
dumpDepsFileObject.write( "# dependency dump of package %s\n" % ( packageName ) )
for info in deplist:
isVCSTarget = False
if args.dumpDepsFile:
dumpDepsFileObject.write( ",".join( [ info.category, info.package, info.target, "" ] ) + "\n" )
isLastPackage = info == deplist[ -1 ]
if args.outDateVCS or (args.outDatePackage and isLastPackage):
isVCSTarget = portage.PortageInstance.getUpdatableVCSTargets( info.category, info.package ) != [ ]
isInstalled = InstallDB.installdb.isInstalled( info.category, info.package )
if args.list_file and action != "all":
info.enabled = info.package in originalPackageList
if ( isInstalled and not info.enabled ) and not (
isInstalled and (args.outDateVCS or (
args.outDatePackage and isLastPackage) ) and isVCSTarget ):
if EmergeDebug.verbose() > 1 and info.package == packageName:
EmergeDebug.warning("already installed %s/%s" % (info.category, info.package))
elif EmergeDebug.verbose() > 2 and not info.package == packageName:
EmergeDebug.warning("already installed %s/%s" % (info.category, info.package))
else:
# in case we only want to see which packages are still to be build, simply return the package name
if args.probe:
if EmergeDebug.verbose() > 0:
EmergeDebug.warning("pretending %s" % info)
else:
if action in [ "install-deps", "update-direct-deps" ]:
action = "all"
if not handlePackage( info.category, info.package, action, args.doContinue, args.update_fast ):
EmergeDebug.error("fatal error: package %s/%s %s failed" % \
( info.category, info.package, action ))
utils.notify( "Emerge build failed",
"Build of %s/%s failed" % ( info.category, info.package),
action )
return False
utils.notify( "Emerge build finished",
"Build of %s/%s finished" % ( info.category, info.package),
action )
EmergeDebug.new_line()
return True
def enterSourceDir(self):
if (not os.path.exists(self.sourceDir())):
return False
EmergeDebug.warning("entering the source directory!")
os.chdir(self.sourceDir())
EmergeDebug.debug("entering: %s" % self.sourceDir())
