def initialize(self):
# Preparse some files
# adding openjre8 version rpm macro
if platform.machine() == "x86_64":
spec = Specutils(constants.specPath + "/openjdk8/openjdk8.spec")
else:
spec = Specutils(constants.specPath +
"/openjdk8/openjdk8_aarch64.spec")
java8version = spec.getVersion()
constants.addMacro("JAVA8_VERSION", java8version)
# adding kernelversion rpm macro
spec = Specutils(constants.specPath + "/linux/linux.spec")
kernelversion = spec.getVersion()
constants.addMacro("KERNEL_VERSION", kernelversion)
# adding kernelrelease rpm macro
kernelrelease = spec.getRelease()
constants.addMacro("KERNEL_RELEASE", kernelrelease)
# adding kernelsubrelease rpm macro
a, b, c = kernelversion.split(".")
kernelsubrelease = (
'%02d%02d%03d%03d' %
(int(a), int(b), int(c), int(kernelrelease.split('.')[0])))
if kernelsubrelease:
kernelsubrelease = "." + kernelsubrelease
constants.addMacro("kernelsubrelease", kernelsubrelease)
# Full parsing
self.specData = SerializableSpecObjectsUtils(constants.logPath)
self.specData.readSpecsAndConvertToSerializableObjects(
constants.specPath)
def buildPackagesList(specPath, csvFilename):
csvFile = open(csvFilename, "w")
csvFile.write("Package,Version,License,URL,Sources,Patches\n")
lst = os.listdir(specPath)
lst.sort()
for dirEntry in lst:
specDir = os.path.join(specPath, dirEntry)
if os.path.isdir(specDir):
for specEntry in os.listdir(specDir):
specFile = os.path.join(specDir, specEntry)
if os.path.isfile(specFile) and specFile.endswith(".spec"):
spec = Specutils(specFile)
name = spec.getBasePackageName()
version = spec.getRPMVersion(name)
license = spec.getLicense(name)
url = spec.getURL(name)
ss = spec.getSourceURLs()
sources = ""
for s in ss:
if (s.startswith("http") or s.startswith("ftp")):
if sources != "":
sources += " "
sources += s
patches = ""
ps = spec.getPatchNames()
for p in ps:
if patches != "":
patches += " "
patches += p
csvFile.write(name + "," + version + "," + license + "," +
url + "," + sources + "," + patches + "\n")
csvFile.close()
def readSpecsAndConvertToSerializableObjects(self, specFilesPath):
listSpecFiles = []
self.getListSpecFiles(listSpecFiles, specFilesPath)
for specFile in listSpecFiles:
skipUpdating = False
spec = Specutils(specFile)
specName = spec.getBasePackageName()
specObj = SerializableSpecObject()
specObj.name = specName
specObj.buildRequirePackages = spec.getBuildRequiresAllPackages()
specObj.installRequiresAllPackages = spec.getRequiresAllPackages()
specObj.listPackages = spec.getPackageNames()
specObj.specFile = specFile
specObj.version = spec.getVersion()
specObj.release = spec.getRelease()
specObj.listSources = spec.getSourceNames()
specObj.checksums = spec.getChecksums()
specObj.listPatches = spec.getPatchNames()
specObj.securityHardening = spec.getSecurityHardeningOption()
for specPkg in specObj.listPackages:
if specPkg in self.mapPackageToSpec:
existingObj = self.mapSerializableSpecObjects[
self.mapPackageToSpec[specPkg]]
if self.compareVersions(existingObj, specObj) == 1:
skipUpdating = True
break
specObj.installRequiresPackages[specPkg] = spec.getRequires(
specPkg)
self.mapPackageToSpec[specPkg] = specName
if skipUpdating == False:
self.mapSerializableSpecObjects[specName] = specObj
def buildSourcesList(specPath, yamlDir, singleFile=False):
strUtils = StringUtils()
if singleFile:
yamlFile = open(yamlDir+"sources_list.yaml", "w")
lst = os.listdir(specPath)
lst.sort()
for dirEntry in lst:
specDir = os.path.join(specPath, dirEntry)
if os.path.isdir(specDir):
for specEntry in os.listdir(specDir):
specFile = os.path.join(specDir, specEntry)
if os.path.isfile(specFile) and specFile.endswith(".spec"):
spec=Specutils(specFile)
modified = len(spec.getPatchNames()) > 0
ss=spec.getSourceURLs()
for s in ss:
if (s.startswith("http") or s.startswith("ftp")):
ossname=strUtils.getPackageNameFromURL(s)
ossversion=strUtils.getPackageVersionFromURL(s)
if not singleFile:
yamlFile = open(yamlDir+ossname+"-"+ossversion+".yaml", "w")
yamlFile.write("vmwsource:"+ossname+":"+ossversion+":\n")
yamlFile.write(" repository: VMWsource\n")
yamlFile.write(" name: '"+ossname+"'\n")
yamlFile.write(" version: '"+ossversion+"'\n")
yamlFile.write(" url: "+s+"\n")
yamlFile.write(" license: UNKNOWN\n")
if modified:
yamlFile.write(" modified: true\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
def buildSourcesList(specPath, sourcePath, yamlDir, logger, singleFile=True):
strUtils = StringUtils()
if singleFile:
yamlFile = open(yamlDir + "sources_list.yaml", "w")
lst = os.listdir(specPath)
lst.sort()
import PullSources
for dirEntry in lst:
specDir = os.path.join(specPath, dirEntry)
if os.path.isdir(specDir):
for specEntry in os.listdir(specDir):
specFile = os.path.join(specDir, specEntry)
if os.path.isfile(specFile) and specFile.endswith(".spec"):
spec = Specutils(specFile)
modified = len(spec.getPatchNames()) > 0
listSourceURLs = spec.getSourceURLs()
ossname = spec.getBasePackageName()
ossversion = spec.getVersion()
url = None
listSourceNames = spec.getSourceNames()
sourceName = None
if len(listSourceNames) > 0:
sourceName = listSourceNames[0]
sha1 = spec.getChecksumForSource(sourceName)
if sha1 is not None:
PullSources.get(sourceName, sha1, sourcePath,
constants.pullsourcesConfig)
if len(listSourceURLs) > 0:
sourceURL = listSourceURLs[0]
if sourceURL.startswith(
"http") or sourceURL.startswith("ftp"):
url = sourceURL
else:
url = spec.getURL(ossname)
if not singleFile:
yamlFile = open(
yamlDir + "/" + ossname + "-" + ossversion +
".yaml", "w")
yamlFile.write("vmwsource:" + ossname + ":" + ossversion +
":\n")
yamlFile.write(" repository: VMWsource\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + ossversion + "'\n")
yamlFile.write(" url: " + str(url) + "\n")
yamlFile.write(" license: UNKNOWN\n")
if sourceName is not None:
yamlFile.write(" vmwsource-distribution: " +
str(sourceName) + "\n")
if modified:
yamlFile.write(" modified: true\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.info("Generated source yaml files for all packages")
def buildSRPMList(specPath, srpmPath, yamlDir, logger, singleFile=True):
strUtils = StringUtils()
if singleFile:
yamlFile = open(yamlDir + "srpm_list.yaml", "w")
lst = os.listdir(specPath)
lst.sort()
cmdUtils = CommandUtils()
for dirEntry in lst:
specDir = os.path.join(specPath, dirEntry)
if os.path.isdir(specDir):
for specEntry in os.listdir(specDir):
specFile = os.path.join(specDir, specEntry)
if os.path.isfile(specFile) and specFile.endswith(".spec"):
spec = Specutils(specFile)
ossname = spec.getBasePackageName()
ossversion = spec.getVersion()
ossrelease = spec.getRelease()
listFoundSRPMFiles = cmdUtils.findFile(
ossname + "-" + ossversion + "-" + ossrelease +
".src.rpm", srpmPath)
srpmName = None
if len(listFoundSRPMFiles) == 1:
srpmFullPath = listFoundSRPMFiles[0]
srpmName = os.path.basename(srpmFullPath)
cpcmd = "cp " + srpmFullPath + " " + yamlDir + "/"
returnVal = cmdUtils.runCommandInShell(cpcmd)
if not returnVal:
logger.error(
"Copy SRPM File is failed for package:" +
ossname)
else:
logger.error("SRPM file is not found:" + ossname)
if not singleFile:
yamlFile = open(
yamlDir + "/" + ossname + "-" + ossversion + "-" +
ossrelease + ".yaml", "w")
yamlFile.write("baseos:" + ossname + ":" + ossversion +
"-" + ossrelease + ":\n")
yamlFile.write(" repository: BaseOS\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + ossversion + "-" +
ossrelease + "'\n")
yamlFile.write(" baseos-style: rpm\n")
yamlFile.write(" baseos-source: '" + str(srpmName) +
"'\n")
yamlFile.write(" baseos-osname: 'photon'\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
logger.info("Generated srpm yaml files for all packages")
def findTotalWhoNeedsToBuild(self, depQue, whoBuildDeps, whoBuildDepSet, displayOption):
while not depQue.empty():
specPkg = depQue.get()
specName = self.getSpecName(specPkg)
spec=Specutils(self.getSpecFile(specPkg))
RPMName=spec.getRPMName(specPkg)
debuginfoRPMName=spec.getDebuginfoRPMName(specPkg)
whoBuildDepSet.add(RPMName)
whoBuildDepSet.add(debuginfoRPMName)
if specName is None:
print specPkg + " is missing"
if not whoBuildDeps.has_key(specPkg):
continue
for depPkg in whoBuildDeps[specPkg]:
depQue.put(depPkg)
def readSpecsAndConvertToSerializableObjects(self, specFilesPath):
listSpecFiles = []
self.getListSpecFiles(listSpecFiles, specFilesPath)
for specFile in listSpecFiles:
spec = Specutils(specFile)
specName = spec.getBasePackageName()
specObj = SpecObject()
specObj.name = specName
specObj.buildRequiresAllPackages = spec.getBuildRequiresAllPackages(
)
specObj.extraBuildRequires = spec.getExtraBuildRequires()
specObj.installRequiresAllPackages = spec.getRequiresAllPackages()
specObj.checkBuildRequirePackages = spec.getCheckBuildRequiresAllPackages(
)
specObj.listPackages = spec.getPackageNames()
specObj.specFile = specFile
specObj.version = spec.getVersion()
specObj.release = spec.getRelease()
specObj.listSources = spec.getSourceNames()
specObj.checksums = spec.getChecksums()
specObj.specDefs = spec.getDefinitions()
specObj.listPatches = spec.getPatchNames()
specObj.securityHardening = spec.getSecurityHardeningOption()
specObj.isCheckAvailable = spec.isCheckAvailable()
specObj.license = spec.getLicense()
specObj.url = spec.getURL()
specObj.sourceurl = spec.getSourceURL()
for specPkg in specObj.listPackages:
specObj.installRequiresPackages[specPkg] = spec.getRequires(
specPkg)
specObj.buildarch[specPkg] = spec.getBuildArch(specPkg)
# TODO add multiversioning support
self.mapPackageToSpec[specPkg] = specName
if spec.getIsRPMPackage(specPkg):
specObj.listRPMPackages.append(specPkg)
if specName in self.mapSpecObjects:
self.mapSpecObjects[specName].append(specObj)
else:
self.mapSpecObjects[specName] = [specObj]
self.mapSpecFileNameToSpecObj[os.path.basename(specFile)] = specObj
for key, value in self.mapSpecObjects.items():
if len(value) > 1:
self.mapSpecObjects[key] = sorted(
value, key=lambda x: self.compareVersions(x), reverse=True)
def buildSourcesList(specPath, yamlDir, singleFile=False):
strUtils = StringUtils()
if singleFile:
yamlFile = open(yamlDir + "sources_list.yaml", "w")
lst = os.listdir(specPath)
lst.sort()
for dirEntry in lst:
specDir = os.path.join(specPath, dirEntry)
if os.path.isdir(specDir):
for specEntry in os.listdir(specDir):
specFile = os.path.join(specDir, specEntry)
if os.path.isfile(specFile) and specFile.endswith(".spec"):
spec = Specutils(specFile)
modified = len(spec.getPatchNames()) > 0
listSourceURLs = spec.getSourceURLs()
ossname = spec.getBasePackageName()
ossversion = spec.getVersion()
url = None
if len(listSourceURLs) > 0:
sourceURL = listSourceURLs[0]
if sourceURL.startswith(
"http") or sourceURL.startswith("ftp"):
url = sourceURL
else:
url = spec.getURL(ossname)
if not singleFile:
yamlFile = open(
yamlDir + ossname + "-" + ossversion + ".yaml",
"w")
yamlFile.write("vmwsource:" + ossname + ":" + ossversion +
":\n")
yamlFile.write(" repository: VMWsource\n")
yamlFile.write(" name: '" + ossname + "'\n")
yamlFile.write(" version: '" + ossversion + "'\n")
yamlFile.write(" url: " + str(url) + "\n")
yamlFile.write(" license: UNKNOWN\n")
if modified:
yamlFile.write(" modified: true\n")
yamlFile.write("\n")
if not singleFile:
yamlFile.close()
if singleFile:
yamlFile.close()
def readSpecsAndConvertToSerializableObjects(self, specFilesPath):
listSpecFiles = []
self.getListSpecFiles(listSpecFiles, specFilesPath)
for specFile in listSpecFiles:
spec = Specutils(specFile)
specName = spec.getBasePackageName()
specObj = SerializableSpecObject()
specObj.name = specName
specObj.buildRequirePackages = spec.getBuildRequiresAllPackages()
specObj.installRequiresAllPackages = spec.getRequiresAllPackages()
specObj.listPackages = spec.getPackageNames()
specObj.specFile = specFile
specObj.version = spec.getVersion()
specObj.release = spec.getRelease()
specObj.listSources = spec.getSourceNames()
specObj.listPatches = spec.getPatchNames()
for specPkg in specObj.listPackages:
specObj.installRequiresPackages[specPkg] = spec.getRequires(
specPkg)
self.mapPackageToSpec[specPkg] = specName
self.mapSerializableSpecObjects[specName] = specObj
def readSpecsAndConvertToSerializableObjects(self, specFilesPath,
inputType, inputValue,
displayOption):
children = {}
listSpecFiles = []
whoNeedsList = []
whoBuildDepSet = set()
independentRPMS = [
] # list of all RPMS not built from photon and that must be blindly copied.
whoBuildDeps = {}
allDeps = {}
parent = {}
depQue = Queue.Queue()
packageFound = False
self.getListSpecFiles(listSpecFiles, specFilesPath)
for specFile in listSpecFiles:
spec = Specutils(specFile)
specName = spec.getBasePackageName()
specObj = SerializableSpecObject()
specObj.name = specName
specObj.buildRequirePackages = spec.getBuildRequiresAllPackages()
specObj.installRequiresAllPackages = spec.getRequiresAllPackages()
specObj.listPackages = spec.getPackageNames()
specObj.specFile = specFile
specObj.version = spec.getVersion()
specObj.release = spec.getRelease()
specObj.listSources = spec.getSourceNames()
specObj.listPatches = spec.getPatchNames()
specObj.securityHardening = spec.getSecurityHardeningOption()
for specPkg in specObj.listPackages:
specObj.installRequiresPackages[specPkg] = spec.getRequires(
specPkg)
if (
inputType == "pkg" and inputValue == specPkg
): # all the first level dependencies to a dictionary and queue
packageFound = True
for depPkg in specObj.installRequiresPackages[specPkg]:
if False == allDeps.has_key(depPkg):
allDeps[depPkg] = 0
parent[depPkg] = ""
depQue.put(depPkg)
elif (
inputType == "who-needs" and
(inputValue in specObj.installRequiresPackages[specPkg])):
whoNeedsList.append(specPkg)
elif (inputType == "who-needs-build"):
for bdrq in specObj.buildRequirePackages:
if (whoBuildDeps.has_key(bdrq)):
whoBuildDeps[bdrq].add(specPkg)
else:
whoBuildDeps[bdrq] = set()
whoBuildDeps[bdrq].add(specPkg)
if (inputValue == specPkg):
packageFound = True
for depPkg in specObj.listPackages:
depQue.put(depPkg)
self.mapPackageToSpec[specPkg] = specName
self.mapSerializableSpecObjects[specName] = specObj
# Generate dependencies for individual packages
if (inputType == "pkg"):
if (packageFound == True):
self.findTotalRequires(allDeps, depQue, parent, displayOption)
else:
print "No spec file builds a package named", inputValue
return
# Generate dependencies for all packages in the given JSON input file
elif (inputType == "json"):
filePath = self.inputDataDir + "/" + inputValue
data = self.get_all_package_names(filePath)
for pkg in data:
if False == allDeps.has_key(pkg):
spName = self.getSpecName(pkg)
if (spName != None):
allDeps[pkg] = 0
parent[pkg] = ""
depQue.put(pkg)
self.findTotalRequires(allDeps, depQue, parent,
displayOption)
else:
independentRPMS.append(pkg)
#Generating the list of packages that requires the given input package at install time
elif (inputType == "who-needs"):
print whoNeedsList
return
#Generating the list of packages that the modified package will affect at build time
elif (inputType == "who-needs-build"):
if (packageFound == True):
self.findTotalWhoNeedsToBuild(depQue, whoBuildDeps,
whoBuildDepSet, displayOption)
print whoBuildDepSet
else:
print "No spec file builds a package named", inputValue
return
# construct the sorted list of all packages (sorted by dependency)
sortedList = []
for elem in sorted(allDeps.items(),
key=operator.itemgetter(1),
reverse=True):
sortedList.append(elem[0])
sortedList.extend(independentRPMS)
# construct all children nodes
if (displayOption == "tree"):
for k, v in parent.iteritems():
children.setdefault(v, []).append(k)
if (inputType == "json"):
print "Dependency Mappings for", inputValue, ":", "\n----------------------------------------------------", children
print "----------------------------------------------------"
if (children.has_key("")):
for child in children[""]:
print child
self.printTree(allDeps, children, child, 1)
for pkg in independentRPMS:
print pkg
print "******************", len(
sortedList), "packages in total ******************"
else:
if (inputType == "pkg" and len(children) > 0):
print "cyclic dependency detected, mappings: \n", children
# To display a flat list of all packages
elif (displayOption == "list"):
print sortedList
# To generate a new JSON file based on given input json file
elif (displayOption == "json" and inputType == "json"):
d = {}
d['packages'] = sortedList
outFilePath = self.jsonFilesOutPath + inputValue
with open(outFilePath, 'wb') as outfile:
json.dump(d, outfile)
return sortedList
def readSpecsAndConvertToSerializableObjects(self, specFilesPath, inputType, inputValue, displayOption):
children = {}
listSpecFiles=[]
whoNeedsList=[]
allDeps={}
parent={}
depQue = Queue.Queue()
packageFound = False
self.getListSpecFiles(listSpecFiles,specFilesPath)
for specFile in listSpecFiles:
spec=Specutils(specFile)
specName=spec.getBasePackageName()
specObj=SerializableSpecObject()
specObj.name=specName
specObj.buildRequirePackages=spec.getBuildRequiresAllPackages()
specObj.installRequiresAllPackages=spec.getRequiresAllPackages()
specObj.listPackages=spec.getPackageNames()
specObj.specFile=specFile
specObj.version=spec.getVersion()
specObj.release=spec.getRelease()
specObj.listSources=spec.getSourceNames()
specObj.listPatches=spec.getPatchNames()
specObj.securityHardening=spec.getSecurityHardeningOption()
for specPkg in specObj.listPackages:
specObj.installRequiresPackages[specPkg]=spec.getRequires(specPkg)
if( inputType == "pkg" and inputValue == specPkg): # all all the first level dependencies to a dictionary and queue
packageFound = True
for depPkg in specObj.installRequiresPackages[specPkg]:
if False == allDeps.has_key(depPkg):
allDeps[depPkg] = 0
parent[depPkg] = ""
depQue.put(depPkg)
if (inputType == "who-needs"):
if (inputValue in specObj.installRequiresPackages[specPkg]):
whoNeedsList.append(specPkg)
self.mapPackageToSpec[specPkg]=specName
self.mapSerializableSpecObjects[specName]=specObj
if (inputType == "pkg"):
if (packageFound == True):
self.findTotalRequires(allDeps, depQue, parent, displayOption)
else:
print "No spec file builds a package named",inputValue
return
elif (inputType == "json"):
filePath = self.inputDataDir +"/"+ inputValue
data = self.get_all_package_names(filePath)
#print data
for pkg in data:
if False == allDeps.has_key(pkg):
allDeps[pkg] = 0
parent[pkg] = ""
depQue.put(pkg)
self.findTotalRequires(allDeps, depQue, parent, displayOption)
elif (inputType == "who-needs"):
print whoNeedsList
# construct the sorted list of all packages (sorted by dependency)
sortedList = []
for elem in sorted(allDeps.items(), key=operator.itemgetter(1), reverse=True):
sortedList.append(elem[0])
# construct all children nodes
if (displayOption == "tree"):
#print "parent:" ,parent
for k, v in parent.iteritems():
children.setdefault(v, []).append(k)
if(inputType == "json"):
print "Dependency Mappings:\n", children
if (children.has_key("")):
for child in children[""]:
print child
self.printTree(allDeps, children, child, 1)
else:
if (inputType == "pkg" and len(children) > 0):
print "cyclic dependency mappings: \n",children
elif(displayOption == "list"):
print sortedList
elif(displayOption == "json" and inputType == "json"):
d = {}
d['packages'] = sortedList
outFilePath = self.jsonFilesOutPath + inputValue
with open(outFilePath, 'wb') as outfile:
json.dump(d, outfile)