def getDependencies(path, name="", verbose=True):
deps = []
if not os.path.isdir(path) or not os.path.exists(
os.path.join(path, "configure")):
return None
if verbose:
logger.writeMessage("Analyzing 'configure --help' output", name)
helpFileName = os.path.join(path, "configure_help.log")
helpFile = open(helpFileName, "w")
try:
returnCode = utilityFunctions.executeSubProcess(
"./configure --help", path, helpFile.fileno())
finally:
helpFile.close()
if returnCode != 0:
return None
try:
helpFile = open(helpFileName, "r")
regexp = re.compile(
r"--with-([a-zA-Z\-_]+)=(?:PREFIX|prefix|PATH|path|DIR|dir)")
for line in helpFile:
match = regexp.search(line)
if match != None:
foundDep = match.group(1)
foundDep = target.normalizeName(foundDep)
if not foundDep in deps:
deps.append(foundDep)
finally:
helpFile.close()
return deps
def validate(self, mdOptions):
normalizedName = normalizeName(self.name)
if normalizedName == "":
return False
if self.path == "":
return False
for alias in self.aliases:
if normalizeName(alias) == normalizedName:
logger.writeError("Target's alias cannot be same as it's name", self.name)
return False
#Check for write access to install directories used in commands.
if mdOptions.targetSpecifiedToBuild(self.name) and not mdOptions.cleanMode and not mdOptions.importMode:
for buildStep in self.buildSteps:
if buildStep.command == "":
continue
expandedCommand = mdOptions.defines.expand(buildStep.command)
installDir = autoTools.getInstallDir(expandedCommand)
if installDir == "":
installDir = cmake.getInstallDir(expandedCommand)
if installDir != "" and not utilityFunctions.haveWriteAccess(installDir):
logger.writeError("No write access to used install directory: " + installDir, self.name, buildStep.name, mdOptions.projectFile)
if not mdOptions.prefixDefined:
logger.writeMessage("Use command-line option '-p<install path>' or running MixDown with superuser privileges (sudo)")
else:
logger.writeMessage("Choose a different install directory for command-line option '-p<install path>'")
return False
return True
def getDependencies(path, name="", verbose=True):
deps = []
if not os.path.isdir(path) or not os.path.exists(os.path.join(path, "configure")):
return None
if verbose:
logger.writeMessage("Analyzing 'configure --help' output", name)
helpFileName = os.path.join(path, "configure_help.log")
helpFile = open(helpFileName, "w")
try:
returnCode = utilityFunctions.executeSubProcess("./configure --help", path, helpFile.fileno())
finally:
helpFile.close()
if returnCode != 0:
return None
try:
helpFile = open(helpFileName, "r")
regexp = re.compile(r"--with-([a-zA-Z\-_]+)=(?:PREFIX|prefix|PATH|path|DIR|dir)")
for line in helpFile:
match = regexp.search(line)
if match != None:
foundDep = match.group(1)
foundDep = target.normalizeName(foundDep)
if not foundDep in deps:
deps.append(foundDep)
finally:
helpFile.close()
return deps
def generateConfigureFiles(path, name, verbose=True):
command = getPreconfigureCommand(path)
if command != "":
if verbose:
logger.writeMessage("Generating build files...", name)
returnCode = utilityFunctions.executeSubProcess(command, path)
if returnCode != 0:
return False
return True
def generateConfigureFiles(path, name, verbose=True):
command = getPreconfigureCommand(path)
if command != "":
if verbose:
logger.writeMessage("Generating build files...", name)
returnCode = utilityFunctions.executeSubProcess(command, path)
if returnCode != 0:
return False
return True
def getDependencies(path, name="", verbose=True):
deps = []
if not os.path.isdir(path):
return None
if not isCMakeProject(path):
return None
if verbose:
logger.writeMessage("Analyzing CMake files for dependencies...", name)
fileList = list()
_findAllCMakeFiles(path, fileList)
packageRegExp = re.compile(r"find_package\s*\((\s*[\w\.]+)\s*.*\)")
#find_library(...) starts with a output variable then has two options
# 1) the name of the library to search for
# 2) "NAMES" followed by a list of possible library names
#It can then be followed by possible paths and various tokens listed below
#By using the first name in the list I should be to avoid any extra paths that may or may not exist and get
# the majority of library names right (since most examples I found were close to "OpenGL, OpenGL3.1, OpenGL, 3.2, etc"
libraryRegExp = re.compile(r"find_library\s*\(\s*([^\)]+)\s*\)")
libraryTokens = ['names', 'hints', 'paths', 'path_suffixes', 'doc', 'no_default_path', 'no_cmake_environment_path', 'no_cmake_path', 'no_system_environment_path', 'no_cmake_system_path', 'cmake_find_root_path_both', 'only_cmake_find_root_path', 'no_cmake_find_root_path']
for name in fileList:
try:
cmakeFile = open(name, "r")
for line in cmakeFile:
match = packageRegExp.search(line)
if match != None:
foundDep = target.normalizeName(match.group(1))
if not foundDep in deps:
deps.append(foundDep)
ignoredFirstParam = False
match = libraryRegExp.search(line)
if match != None:
paramStr = match.group(1).strip()
paramList = paramStr.split(" ")
for param in paramList:
param = target.normalizeName(param)
if param != "":
if not ignoredFirstParam:
ignoredFirstParam = True
continue
if param in libraryTokens:
continue
foundDep = param
break
if not foundDep.startswith("${") and not foundDep in deps:
deps.append(foundDep)
ignoredFirstParam = False
finally:
cmakeFile.close()
return deps
def searchForPossibleAliasInList(possibleAlias, targetList, interactive=False):
for target in targetList:
if possibleAlias == target.name or possibleAlias in target.aliases:
return target
elif target.name.startswith(possibleAlias):
if interactive:
userInput = raw_input("Is " + possibleAlias + " an alias for " + target.name + "? ").lower()
if userInput == "y" or userInput == "yes":
target.aliases.append(possibleAlias)
return target
else:
logger.writeMessage("Alias added (" + possibleAlias + ")", target.name)
target.aliases.append(possibleAlias)
return target
return None
def isHgInstalled():
global _isHgInstalled
if _isHgInstalled == None:
outFile = open(os.devnull, "w")
try:
returnCode = utilityFunctions.executeSubProcess("hg --help", outFileHandle = outFile)
except:
#Assume any exceptions means Hg is not installed
returnCode = 1
outFile.close()
if returnCode == 0:
_isHgInstalled = True
else:
logger.writeMessage("Hg is not installed, hg repositories will fail to be checked out")
_isHgInstalled = False
return _isHgInstalled
def isGitInstalled():
global _isGitInstalled
if _isGitInstalled == None:
outFile = open(os.devnull, "w")
try:
returnCode = utilityFunctions.executeSubProcess("git --help", outFileHandle = outFile)
except:
#Assume any exceptions means Git is not installed
returnCode = 1
outFile.close()
if returnCode == 0:
_isGitInstalled = True
else:
logger.writeMessage("Git is not installed, git repositories will fail to be checked out")
_isGitInstalled = False
return _isGitInstalled
def addSkipStepFromOptions(self, options):
if options.skipSteps == '' or options.skipSteps == None:
return True
skipSteps = options.skipSteps
for item in skipSteps.split(","):
pair = item.split(":")
if len(pair) != 2 or pair[0] == "" or pair[1] == "":
logger.writeError("Invalid command-line -s pair found: " + item)
logger.writeMessage("Proper use: -s[Semi-colon delimited list of Skip Step pairs]")
logger.writeMessage("Skip Step Pair: [targetName]:[Steps to skip, comma delimited]")
return False
targets = self.getTarget(pair[0])
if targets == None:
logger.writeError("Target not found in -s command-line option: " + pair[0])
return False
targets.skipSteps = pair[1].split(",")
return True
def searchForPossibleAliasInList(possibleAlias, targetList, interactive=False):
for target in targetList:
if possibleAlias == target.name or possibleAlias in target.aliases:
return target
elif target.name.startswith(possibleAlias):
if interactive:
userInput = raw_input("Is " + possibleAlias +
" an alias for " + target.name +
"? ").lower()
if userInput == "y" or userInput == "yes":
target.aliases.append(possibleAlias)
return target
else:
logger.writeMessage("Alias added (" + possibleAlias + ")",
target.name)
target.aliases.append(possibleAlias)
return target
return None
def addSkipStepFromOptions(self, options):
if options.skipSteps == '' or options.skipSteps == None:
return True
skipSteps = options.skipSteps
for item in skipSteps.split(","):
pair = item.split(":")
if len(pair) != 2 or pair[0] == "" or pair[1] == "":
logger.writeError("Invalid command-line -s pair found: " +
item)
logger.writeMessage(
"Proper use: -s[Semi-colon delimited list of Skip Step pairs]"
)
logger.writeMessage(
"Skip Step Pair: [targetName]:[Steps to skip, comma delimited]"
)
return False
targets = self.getTarget(pair[0])
if targets == None:
logger.writeError(
"Target not found in -s command-line option: " + pair[0])
return False
targets.skipSteps = pair[1].split(",")
return True
def buildStepActor(target, options, project, lock=None):
if target.isStepToBeSkipped(target.currBuildStep.name):
try:
if lock:
lock.acquire()
logger.reportSkipped(target.name, target.currBuildStep.name, "Target specified to skip step")
finally:
if lock:
lock.release()
return True
if target.isStepPreviouslyDone(target.currBuildStep.name):
try:
if lock:
lock.acquire()
logger.reportSkipped(
target.name, target.currBuildStep.name, "Build step successfully built in previous MixDown build"
)
finally:
if lock:
lock.release()
return True
try:
if lock:
lock.acquire()
logger.reportStart(target.name, target.currBuildStep.name)
# Refresh defines before start of every step
project.setTargetFieldsAsDefines(options.defines)
finally:
if lock:
lock.release()
returnCode = None
timeStart = time.time()
command = options.defines.expand(target.currBuildStep.command)
isPythonCommand, namespace, function = python.parsePythonCommand(command)
if isPythonCommand:
success = python.callPythonCommand(namespace, function, target, options, project)
if not success:
returnCode = 1
else:
returnCode = 0
else:
try:
if lock:
lock.acquire()
logger.writeMessage("Executing command: " + command, target.name, target.currBuildStep.name, True)
finally:
if lock:
lock.release()
if not os.path.exists(target.path):
logger.writeError(
target.name
+ "'s path does not exist when about to execute build command in step "
+ target.currBuildStep.name
+ ".",
filePath=target.path,
)
returnCode = 1
else:
outFd = logger.getOutFd(target.name, target.currBuildStep.name)
returnCode = utilityFunctions.executeSubProcess(command, target.path, outFd)
timeFinished = time.time()
timeElapsed = timeFinished - timeStart
if returnCode != 0:
target.currBuildStep.success = False
try:
if lock:
lock.acquire()
logger.reportFailure(target.name, target.currBuildStep.name, timeElapsed, returnCode)
finally:
if lock:
lock.release()
return False
target.currBuildStep.success = True
try:
if lock:
lock.acquire()
logger.reportSuccess(target.name, target.currBuildStep.name, timeElapsed)
finally:
if lock:
lock.release()
return True
def importTargets(options):
logger.setLogger("console")
finalTargets = []
ignoredTargets = []
partialImport = False
fetchStep = commands.BuildStep("fetch", commands.__getFetchCommand(None))
unpackStep = commands.BuildStep("unpack", commands.__getUnpackCommand(None))
tempDir = tempfile.mkdtemp(prefix="mixdown-")
options.downloadDir = os.path.join(tempDir, "mdDownloads")
while len(options.targetsToImport) != 0:
target = options.targetsToImport.pop(0)
logger.writeMessage("Analyzing target...", target.name)
logger.writeMessage("Extracting target...", target.name)
target.outputPath = os.path.join(tempDir, target.name)
target.currBuildStep = fetchStep
if not commands.buildStepActor(target, options, None):
utilityFunctions.removeDir(tempDir)
return None, False
target.currBuildStep = unpackStep
if not commands.buildStepActor(target, options, None):
utilityFunctions.removeDir(tempDir)
return None, False
#Generate build files and find possible dependencies
possibleDeps = []
if cmake.isCMakeProject(target.path):
logger.writeMessage("CMake project found...", target.name)
logger.writeMessage("Analyzing for dependencies...", target.name)
possibleDeps = cmake.getDependencies(target.path, target.name)
elif autoTools.isAutoToolsProject(target.path):
logger.writeMessage("Auto Tools project found...", target.name)
if not os.path.exists(os.path.join(target.path, "configure")):
if not autoTools.generateConfigureFiles(target.path, target.name):
utilityFunctions.removeDir(tempDir)
return None, False
logger.writeMessage("Analyzing for dependencies...", target.name)
possibleDeps = autoTools.getDependencies(target.path, target.name)
if possibleDeps == None:
target.comment = "Unable to parse 'configure --help' output. MixDown cannot determine dependencies for this target."
logger.writeError(target.comment, target.name)
partialImport = True
possibleDeps = []
elif make.isMakeProject(target.path):
target.comment = "Make project found. MixDown cannot determine dependencies from Make projects."
logger.writeError(target.comment, target.name)
partialImport = True
else:
target.comment = "Unknown build system found. MixDown cannot determine dependencies or build commands."
logger.writeError(target.comment, target.name)
partialImport = True
#Find actual dependencies
for possibleDependency in possibleDeps:
if getTarget(possibleDependency, finalTargets + options.targetsToImport):
logger.writeMessage("Known dependency found (" + possibleDependency + ")", target.name)
target.dependsOn.append(possibleDependency)
continue
elif options.interactive and possibleDependency in ignoredTargets:
logger.writeMessage("Previously ignored dependency found (" + possibleDependency + ")", target.name)
continue
if searchForPossibleAliasInList(possibleDependency, finalTargets + options.targetsToImport, options.interactive):
target.dependsOn.append(possibleDependency)
elif not options.interactive:
logger.writeMessage("Ignoring unknown dependency (" + possibleDependency + ")", target.name)
else:
logger.writeMessage("Unknown dependency found (" + possibleDependency + ")", target.name)
userInput = raw_input(possibleDependency + ": Input location, target name, or blank to ignore:").strip()
if userInput == "":
ignoredTargets.append(possibleDependency)
elif os.path.isfile(userInput) or os.path.isdir(userInput) or utilityFunctions.isURL(userInput):
name = target.targetPathToName(userInput)
if name == "":
return None, False
newTarget = target.Target(name, userInput)
options.targetsToImport.append(newTarget)
if target.normalizeName(possibleDependency) != target.normalizeName(userInput):
newTarget.aliases.append(possibleDependency)
target.dependsOn.append(possibleDependency)
else:
aliasTarget = getTarget(userInput, finalTargets + options.targetsToImport, possibleDependency)
if aliasTarget != None:
logger.writeMessage("Alias added (" + userInput + ")", aliasTarget.name)
target.dependsOn.append(possibleDependency)
else:
aliasLocation = raw_input(userInput + ": Target name not found in any known targets. Location of new target:").strip()
if os.path.isfile(aliasLocation) or os.path.isdir(aliasLocation) or utilityFunctions.isURL(aliasLocation):
name = target.targetPathToName(aliasLocation)
if name == "":
return None, False
newTarget = target.Target(name, aliasLocation)
notReviewedTargets.append(newTarget)
if target.normalizeName(possibleDependency) != target.normalizeName(aliasLocation):
newTarget.aliases.append(possibleDependency)
target.dependsOn.append(possibleDependency)
else:
logger.writeError(userInput + ": Alias location not understood.", exitProgram=True)
finalTargets.append(target)
#Create project for targets
projects = project.Project("ProjectNameNotDetermined", finalTargets)
if not projects.examine(options):
logger.writeError("Project failed examination", exitProgram=True)
if not projects.validate(options):
logger.writeError("Project failed validation", exitProgram=True)
mainTargetPath = projects.targets[0].origPath
if utilityFunctions.isURL(mainTargetPath):
mainTargetPath = utilityFunctions.URLToFilename(mainTargetPath)
mainTargetName, mainTargetVersion = utilityFunctions.splitFileName(mainTargetPath)
if mainTargetVersion != "":
projects.name = mainTargetName + "-" + mainTargetVersion
else:
projects.name = mainTargetName
projects.path = projects.name + ".md"
for target in projects.targets:
target.outputPath = ""
if projects.examine(options):
logger.writeMessage("\nFinal targets...\n\n" + str(projects))
projects.write()
utilityFunctions.removeDir(tempDir)
return projects, partialImport
def importTargets(options):
logger.setLogger("console")
finalTargets = []
ignoredTargets = []
partialImport = False
fetchStep = commands.BuildStep("fetch", commands.__getFetchCommand(None))
unpackStep = commands.BuildStep("unpack",
commands.__getUnpackCommand(None))
tempDir = tempfile.mkdtemp(prefix="mixdown-")
options.downloadDir = os.path.join(tempDir, "mdDownloads")
while len(options.targetsToImport) != 0:
target = options.targetsToImport.pop(0)
logger.writeMessage("Analyzing target...", target.name)
logger.writeMessage("Extracting target...", target.name)
target.outputPath = os.path.join(tempDir, target.name)
target.currBuildStep = fetchStep
if not commands.buildStepActor(target, options, None):
utilityFunctions.removeDir(tempDir)
return None, False
target.currBuildStep = unpackStep
if not commands.buildStepActor(target, options, None):
utilityFunctions.removeDir(tempDir)
return None, False
#Generate build files and find possible dependencies
possibleDeps = []
if cmake.isCMakeProject(target.path):
logger.writeMessage("CMake project found...", target.name)
logger.writeMessage("Analyzing for dependencies...", target.name)
possibleDeps = cmake.getDependencies(target.path, target.name)
elif autoTools.isAutoToolsProject(target.path):
logger.writeMessage("Auto Tools project found...", target.name)
if not os.path.exists(os.path.join(target.path, "configure")):
if not autoTools.generateConfigureFiles(
target.path, target.name):
utilityFunctions.removeDir(tempDir)
return None, False
logger.writeMessage("Analyzing for dependencies...", target.name)
possibleDeps = autoTools.getDependencies(target.path, target.name)
if possibleDeps == None:
target.comment = "Unable to parse 'configure --help' output. MixDown cannot determine dependencies for this target."
logger.writeError(target.comment, target.name)
partialImport = True
possibleDeps = []
elif make.isMakeProject(target.path):
target.comment = "Make project found. MixDown cannot determine dependencies from Make projects."
logger.writeError(target.comment, target.name)
partialImport = True
else:
target.comment = "Unknown build system found. MixDown cannot determine dependencies or build commands."
logger.writeError(target.comment, target.name)
partialImport = True
#Find actual dependencies
for possibleDependency in possibleDeps:
if getTarget(possibleDependency,
finalTargets + options.targetsToImport):
logger.writeMessage(
"Known dependency found (" + possibleDependency + ")",
target.name)
target.dependsOn.append(possibleDependency)
continue
elif options.interactive and possibleDependency in ignoredTargets:
logger.writeMessage(
"Previously ignored dependency found (" +
possibleDependency + ")", target.name)
continue
if searchForPossibleAliasInList(
possibleDependency, finalTargets + options.targetsToImport,
options.interactive):
target.dependsOn.append(possibleDependency)
elif not options.interactive:
logger.writeMessage(
"Ignoring unknown dependency (" + possibleDependency + ")",
target.name)
else:
logger.writeMessage(
"Unknown dependency found (" + possibleDependency + ")",
target.name)
userInput = raw_input(
possibleDependency +
": Input location, target name, or blank to ignore:"
).strip()
if userInput == "":
ignoredTargets.append(possibleDependency)
elif os.path.isfile(userInput) or os.path.isdir(
userInput) or utilityFunctions.isURL(userInput):
name = target.targetPathToName(userInput)
if name == "":
return None, False
newTarget = target.Target(name, userInput)
options.targetsToImport.append(newTarget)
if target.normalizeName(
possibleDependency) != target.normalizeName(
userInput):
newTarget.aliases.append(possibleDependency)
target.dependsOn.append(possibleDependency)
else:
aliasTarget = getTarget(
userInput, finalTargets + options.targetsToImport,
possibleDependency)
if aliasTarget != None:
logger.writeMessage("Alias added (" + userInput + ")",
aliasTarget.name)
target.dependsOn.append(possibleDependency)
else:
aliasLocation = raw_input(
userInput +
": Target name not found in any known targets. Location of new target:"
).strip()
if os.path.isfile(aliasLocation) or os.path.isdir(
aliasLocation) or utilityFunctions.isURL(
aliasLocation):
name = target.targetPathToName(aliasLocation)
if name == "":
return None, False
newTarget = target.Target(name, aliasLocation)
notReviewedTargets.append(newTarget)
if target.normalizeName(possibleDependency
) != target.normalizeName(
aliasLocation):
newTarget.aliases.append(possibleDependency)
target.dependsOn.append(possibleDependency)
else:
logger.writeError(
userInput + ": Alias location not understood.",
exitProgram=True)
finalTargets.append(target)
#Create project for targets
projects = project.Project("ProjectNameNotDetermined", finalTargets)
if not projects.examine(options):
logger.writeError("Project failed examination", exitProgram=True)
if not projects.validate(options):
logger.writeError("Project failed validation", exitProgram=True)
mainTargetPath = projects.targets[0].origPath
if utilityFunctions.isURL(mainTargetPath):
mainTargetPath = utilityFunctions.URLToFilename(mainTargetPath)
mainTargetName, mainTargetVersion = utilityFunctions.splitFileName(
mainTargetPath)
if mainTargetVersion != "":
projects.name = mainTargetName + "-" + mainTargetVersion
else:
projects.name = mainTargetName
projects.path = projects.name + ".md"
for target in projects.targets:
target.outputPath = ""
if projects.examine(options):
logger.writeMessage("\nFinal targets...\n\n" + str(projects))
projects.write()
utilityFunctions.removeDir(tempDir)
return projects, partialImport