def validate(self, mdOptions):
normalizedName = normalizeName(self.name)
if normalizedName == "":
return False
if self.path == "":
return False
for alias in self.aliases:
if normalizeName(alias) == normalizedName:
logger.writeError("Target's alias cannot be same as it's name", self.name)
return False
#Check for write access to install directories used in commands.
if mdOptions.targetSpecifiedToBuild(self.name) and not mdOptions.cleanMode and not mdOptions.importMode:
for buildStep in self.buildSteps:
if buildStep.command == "":
continue
expandedCommand = mdOptions.defines.expand(buildStep.command)
installDir = autoTools.getInstallDir(expandedCommand)
if installDir == "":
installDir = cmake.getInstallDir(expandedCommand)
if installDir != "" and not utilityFunctions.haveWriteAccess(installDir):
logger.writeError("No write access to used install directory: " + installDir, self.name, buildStep.name, mdOptions.projectFile)
if not mdOptions.prefixDefined:
logger.writeMessage("Use command-line option '-p<install path>' or running MixDown with superuser privileges (sudo)")
else:
logger.writeMessage("Choose a different install directory for command-line option '-p<install path>'")
return False
return True
def _getToken(self, line, previousToken):
token = Token()
start = 0
finish = 0
lineLength = len(line)
if previousToken.type == TokenType.Symbol and previousToken.value == "=":
token.type = TokenType.String
token.value = line
line = ""
elif self._isValidIdentifierCharacter(line[0]):
token.type = TokenType.Identifier
finish = 1
while finish < lineLength:
if not self._isValidIdentifierCharacter(line[finish]):
break
finish += 1
token.value = line[0:finish]
line = line[finish:]
elif line[0] in symbols:
token.type = TokenType.Symbol
token.value = line[0]
line = line[1:]
else:
logger.writeError("Tokenizer failed on: " + line, filePath=self.fileName)
return None, line
line = line.lstrip()
return token, line
def targetPathToName(path):
name = ""
path = path.strip()
if git.isGitRepo(path):
if os.path.isdir(path):
name = os.path.basename(path)
elif utilityFunctions.isURL(path):
name = utilityFunctions.URLToFilename(path)
if name.endswith(".git"):
name = name[:-4]
elif svn.isSvnRepo(path):
if path.endswith(os.sep):
path = path[:-1]
if path.endswith("/trunk") or path.endswith("\trunk"):
path = path[:-6]
if os.path.isdir(path):
name = os.path.basename(path)
elif utilityFunctions.isURL(path):
name = utilityFunctions.URLToFilename(path)
elif utilityFunctions.isURL(path):
name = utilityFunctions.URLToFilename(path)
name = utilityFunctions.splitFileName(name)[0]
elif os.path.isfile(path) and utilityFunctions.validateCompressedFile(path, logger):
name = utilityFunctions.splitFileName(path)[0]
elif os.path.isdir(path):
if path.endswith(os.sep):
name = os.path.basename(path[:-1])
else:
name = os.path.basename(path)
else:
logger.writeError("Could not convert given target path to name: " + path)
return name
def _getToken(self, line, previousToken):
token = Token()
start = 0
finish = 0
lineLength = len(line)
if previousToken.type == TokenType.Symbol and previousToken.value == "=":
token.type = TokenType.String
token.value = line
line = ""
elif self._isValidIdentifierCharacter(line[0]):
token.type = TokenType.Identifier
finish = 1
while finish < lineLength:
if not self._isValidIdentifierCharacter(line[finish]):
break
finish += 1
token.value = line[0:finish]
line = line[finish:]
elif line[0] in symbols:
token.type = TokenType.Symbol
token.value = line[0]
line = line[1:]
else:
logger.writeError("Tokenizer failed on: " + line, filePath=self.fileName)
return None, line
line = line.lstrip()
return token, line
def __validateOptionsDir(self, path):
if os.path.isfile(self.buildDir):
logger.writeError(
"Cannot create directory by MixDown, a file by the same name already exists: "
+ path)
return False
return True
def __assureEqualsInStatusLog(self, l, name, value):
#Note just get rid of this function.....
if l[0].lower() != name.lower():
logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath")
return False
if l[1] != value:
return False
return True
def __assureEqualsInStatusLog(self, l, name, value):
#Note just get rid of this function.....
if l[0].lower() != name.lower():
logger.writeError("Unknown line in status log:\n" + ':'.join(l),
filePath="options.statusLogPath")
return False
if l[1] != value:
return False
return True
def __addTarget(self, targets, lineCount=0):
if targets.name == "" or targets.path == "":
logger.writeError("New target started before previous was finished, all targets require atleast 'Name' and 'Path' to be declared", "", "", self.path, lineCount)
return False
for currTarget in self.targets:
if target.normalizeName(targets.name) == target.normalizeName(currTarget.name):
logger.writeError("Cannot have more than one project target by the same name", currTarget.name, "", self.path, lineCount)
return False
self.targets.append(targets)
return True
def __assignDepthToTargetList(self):
q = Queue.Queue()
for targets in self.targets:
q.put(targets.name)
while not q.empty():
currName = q.get()
currTarget = self.getTarget(currName)
for currChildName in currTarget.dependsOn:
currChildTarget = self.getTarget(currChildName)
if not currChildTarget:
logger.writeError("Target '" + currTarget.name + "' has non-existant dependency '" + currChildName + "'", "", self.path, exitProgram=True)
if currChildTarget.dependencyDepth < (currTarget.dependencyDepth + 1):
currChildTarget.dependencyDepth = currTarget.dependencyDepth + 1
q.put(currChildName)
def determineOutputPath(self, mdOptions):
if self.outputPathSpecified and self.outputPath != "":
return self.outputPath
else:
targetsBuildDir = os.path.join(mdOptions.buildDir, self.name)
if mdOptions.cleanMode:
if os.path.exists(targetsBuildDir) and os.path.isdir(targetsBuildDir):
return targetsBuildDir
elif os.path.isdir(self.path):
return self.path
else:
logger.writeError("Output path could not be located, define in project file with \"output=<path>\"", self.name, "clean")
return ""
return targetsBuildDir
def examine(self, options):
if self.__examined:
return True
else:
if len(self.targets) < 1:
logger.writeError("Project has no targets")
return False
self.__assignDepthToTargetList()
self.targets = self.__sortTargetList(self.targets)
self.__expandDependancyLists()
for targets in self.targets:
if not targets.examine(options):
return False
self.__examined = True
return True
def examine(self, options):
if self.__examined:
return True
else:
if len(self.targets) < 1:
logger.writeError("Project has no targets")
return False
self.__assignDepthToTargetList()
self.targets = self.__sortTargetList(self.targets)
self.__expandDependancyLists()
for targets in self.targets:
if not targets.examine(options):
return False
self.__examined = True
return True
def __addTarget(self, targets, lineCount=0):
if targets.name == "" or targets.path == "":
logger.writeError(
"New target started before previous was finished, all targets require atleast 'Name' and 'Path' to be declared",
"", "", self.path, lineCount)
return False
for currTarget in self.targets:
if target.normalizeName(targets.name) == target.normalizeName(
currTarget.name):
logger.writeError(
"Cannot have more than one project target by the same name",
currTarget.name, "", self.path, lineCount)
return False
self.targets.append(targets)
return True
def addSkipStepFromOptions(self, options):
if options.skipSteps == '' or options.skipSteps == None:
return True
skipSteps = options.skipSteps
for item in skipSteps.split(","):
pair = item.split(":")
if len(pair) != 2 or pair[0] == "" or pair[1] == "":
logger.writeError("Invalid command-line -s pair found: " + item)
logger.writeMessage("Proper use: -s[Semi-colon delimited list of Skip Step pairs]")
logger.writeMessage("Skip Step Pair: [targetName]:[Steps to skip, comma delimited]")
return False
targets = self.getTarget(pair[0])
if targets == None:
logger.writeError("Target not found in -s command-line option: " + pair[0])
return False
targets.skipSteps = pair[1].split(",")
return True
def callPythonCommand(namespace, function, target, options, project):
filename = namespace + ".py"
projectPath = os.path.abspath(".")
if not projectPath in sys.path:
sys.path.insert(0, projectPath)
mixDownPath = os.path.dirname(sys.argv[0])
mdPath = os.path.join(mixDownPath, "md")
if os.path.exists(mdPath) and not mdPath in sys.path:
sys.path.insert(0, mdPath)
stepsPath = os.path.dirname(os.path.abspath(steps.__file__))
if mdPath != stepsPath and not stepsPath in sys.path:
sys.path.insert(0, stepsPath)
namespaceFound = False
for currPath in sys.path:
currNamespacePath = os.path.join(currPath, filename)
if os.path.exists(
currNamespacePath) and not os.path.isdir(currNamespacePath):
namespaceFound = True
break
if not namespaceFound:
logger.writeError("Could not find python file in path.",
filePath=filename)
return False
importedNamespace = __import__(namespace)
try:
target.pythonCallInfo.success = False
target.pythonCallInfo.target = target
target.pythonCallInfo.options = options
target.pythonCallInfo.project = project
pythonCallInfo = getattr(importedNamespace,
function)(target.pythonCallInfo)
except AttributeError as e:
logger.writeError(str(e))
traceback.print_exc(file=logger.getErrorFd())
return False
if not pythonCallInfo.success:
return False
target = pythonCallInfo.target
options = pythonCallInfo.options
project = pythonCallInfo.project
return True
def callPythonCommand(namespace, function, target, options, project):
filename = namespace + ".py"
projectPath = os.path.abspath(".")
if not projectPath in sys.path:
sys.path.insert(0, projectPath)
mixDownPath = os.path.dirname(sys.argv[0])
mdPath = os.path.join(mixDownPath, "md")
if os.path.exists(mdPath) and not mdPath in sys.path:
sys.path.insert(0, mdPath)
stepsPath = os.path.dirname(os.path.abspath(steps.__file__))
if mdPath != stepsPath and not stepsPath in sys.path:
sys.path.insert(0, stepsPath)
namespaceFound = False
for currPath in sys.path:
currNamespacePath = os.path.join(currPath, filename)
if os.path.exists(currNamespacePath) and not os.path.isdir(currNamespacePath):
namespaceFound = True
break
if not namespaceFound:
logger.writeError("Could not find python file in path.", filePath=filename)
return False
importedNamespace = __import__(namespace)
try:
target.pythonCallInfo.success = False
target.pythonCallInfo.target = target
target.pythonCallInfo.options = options
target.pythonCallInfo.project = project
pythonCallInfo = getattr(importedNamespace, function)(target.pythonCallInfo)
except AttributeError as e:
logger.writeError(str(e))
traceback.print_exc(file=logger.getErrorFd())
return False
if not pythonCallInfo.success:
return False
target = pythonCallInfo.target
options = pythonCallInfo.options
project = pythonCallInfo.project
return True
def __assignDepthToTargetList(self):
q = Queue.Queue()
for targets in self.targets:
q.put(targets.name)
while not q.empty():
currName = q.get()
currTarget = self.getTarget(currName)
for currChildName in currTarget.dependsOn:
currChildTarget = self.getTarget(currChildName)
if not currChildTarget:
logger.writeError("Target '" + currTarget.name +
"' has non-existant dependency '" +
currChildName + "'",
"",
self.path,
exitProgram=True)
if currChildTarget.dependencyDepth < (
currTarget.dependencyDepth + 1):
currChildTarget.dependencyDepth = currTarget.dependencyDepth + 1
q.put(currChildName)
def __validateDependsOnLists(self):
if len(self.targets) == 0:
return True
for currTarget in self.targets:
normalizedName = target.normalizeName(currTarget.name)
checkedDependencies = []
for dependency in currTarget.dependsOn:
normalizedDepedancy = target.normalizeName(dependency)
if normalizedDepedancy == normalizedName:
logger.writeError("Target cannot depend on itself",
currTarget.name, "", self.path)
return False
if normalizedDepedancy in checkedDependencies:
logger.writeError(
"Target has duplicate dependency '" + dependency + "'",
currTarget.name, "", self.path)
return False
if self.getTarget(dependency) is None:
logger.writeError(
"Target has non-existant dependency '" + dependency +
"'", currTarget.name, "", self.path)
return False
checkedDependencies.append(normalizedDepedancy)
path = [target.normalizeName(self.targets[0].name)]
return self.__searchPathsForCycles(path)
def addSkipStepFromOptions(self, options):
if options.skipSteps == '' or options.skipSteps == None:
return True
skipSteps = options.skipSteps
for item in skipSteps.split(","):
pair = item.split(":")
if len(pair) != 2 or pair[0] == "" or pair[1] == "":
logger.writeError("Invalid command-line -s pair found: " +
item)
logger.writeMessage(
"Proper use: -s[Semi-colon delimited list of Skip Step pairs]"
)
logger.writeMessage(
"Skip Step Pair: [targetName]:[Steps to skip, comma delimited]"
)
return False
targets = self.getTarget(pair[0])
if targets == None:
logger.writeError(
"Target not found in -s command-line option: " + pair[0])
return False
targets.skipSteps = pair[1].split(",")
return True
def __processProfileCommandline(self, commandline):
if len(commandline) < 2:
self.printUsage()
return False
self.verbose = True
for currArg in commandline[1:]:
if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"):
self.printUsage()
return False
elif currArg == "--profile":
continue
currFlag = currArg[:2].lower()
currValue = currArg[2:]
if currFlag in ("-p", "-l", "-n", "-j", "-b", "-w", "-k", "-o",
"-g", "-i", "-v", "-f"):
logger.writeError(
"Command-line option is not allowed in profile mode: " +
currArg)
return False
elif currFlag == "-d":
if not validateOptionPair(currFlag, currValue):
return False
paths = currValue.split(",")
for path in paths:
self.overrideSearchPath.append((path, False))
elif currFlag == "-r":
if not validateOptionPair(currFlag, currValue):
return False
paths = currValue.split(",")
for path in paths:
self.overrideSearchPath.append((path, True))
elif os.path.splitext(currArg)[1] == ".mdo":
if os.path.isfile(currArg):
logger.writeError(
"Given override file name already exists: " + currArg)
return False
self.overrideFile = currArg
else:
logger.writeError(
"File not found or command-line option not understood: " +
currArg)
return False
if len(self.overrideSearchPath) == 0:
self.overrideSearchPath = [("/usr/local/bin", False),
("/usr/bin", False)]
if self.overrideFile == "":
self.overrideFile = os.path.join(os.getcwd(),
socket.gethostname() + ".mdo")
return True
def __processImportCommandline(self, commandline):
if len(commandline) < 3:
self.printUsage()
return False
for currArg in commandline[1:]:
if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"):
self.printUsage()
return False
elif currArg == "--import":
continue
elif utilityFunctions.isURL(currArg) or os.path.isfile(
currArg) or os.path.isdir(currArg):
name = target.targetPathToName(currArg)
if name == "":
return False
currTarget = target.Target(name, currArg)
self.targetsToImport.append(currTarget)
continue
currFlag = currArg[:2].lower()
currValue = currArg[2:]
if currFlag in ("-p", "-l", "-n", "-j", "-b", "-w", "-k", "-o",
"-g", "-f"):
logger.writeError(
"Command-line option is not allowed in import mode: " +
currArg)
return False
if currFlag == "-i":
if not validateOption(currFlag, currValue):
return False
self.interactive = True
elif currFlag == "-v":
if not validateOption(currFlag, currValue):
return False
self.verbose = True
else:
logger.writeError(
"File not found or command-line option not understood: " +
currArg)
return False
if len(self.targetsToImport) == 0:
self.printUsage()
logger.writeError(
"No packages given when MixDown is in import mode")
return False
return True
def validate(self):
if not self.__validateOptionsDir(self.buildDir) or not\
self.__validateOptionsDir(self.downloadDir) or not\
self.__validateOptionsDir(self.logDir):
return False
if self.importMode and self.cleanMode:
logger.writeError("MixDown cannot be in both import mode and clean mode")
return False
if self.overrideFile == "" and len(self.overrideGroupNames) == 0:
logger.writeError("Override group name(s) given but no override file specified. Use command-line option '-o<filename>'.")
return False
if self.overrideFile != "" and len(self.overrideGroupNames) == 0:
logger.writeError("Override file given but no override group(s) specified. Use command-line option '-g<Name>[,<Name>...]'.")
return False
return True
def __processProfileCommandline(self, commandline):
if len(commandline) < 2:
self.printUsage()
return False
self.verbose = True
for currArg in commandline[1:]:
if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"):
self.printUsage()
return False
elif currArg == "--profile":
continue
currFlag = currArg[:2].lower()
currValue = currArg[2:]
if currFlag in ("-p", "-l", "-n", "-j", "-b", "-w", "-k", "-o", "-g", "-i", "-v", "-f"):
logger.writeError("Command-line option is not allowed in profile mode: " + currArg)
return False
elif currFlag == "-d":
if not validateOptionPair(currFlag, currValue):
return False
paths = currValue.split(",")
for path in paths:
self.overrideSearchPath.append((path, False))
elif currFlag == "-r":
if not validateOptionPair(currFlag, currValue):
return False
paths = currValue.split(",")
for path in paths:
self.overrideSearchPath.append((path, True))
elif os.path.splitext(currArg)[1] == ".mdo":
if os.path.isfile(currArg):
logger.writeError("Given override file name already exists: " + currArg)
return False
self.overrideFile = currArg
else:
logger.writeError("File not found or command-line option not understood: " + currArg)
return False
if len(self.overrideSearchPath) == 0:
self.overrideSearchPath = [("/usr/local/bin", False), ("/usr/bin", False)]
if self.overrideFile == "":
self.overrideFile = os.path.join(os.getcwd(), socket.gethostname() + ".mdo")
return True
def __processImportCommandline(self, commandline):
if len(commandline) < 3:
self.printUsage()
return False
for currArg in commandline[1:]:
if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"):
self.printUsage()
return False
elif currArg == "--import":
continue
elif utilityFunctions.isURL(currArg) or os.path.isfile(currArg) or os.path.isdir(currArg):
name = target.targetPathToName(currArg)
if name == "":
return False
currTarget = target.Target(name, currArg)
self.targetsToImport.append(currTarget)
continue
currFlag = currArg[:2].lower()
currValue = currArg[2:]
if currFlag in ("-p", "-l", "-n", "-j", "-b", "-w", "-k", "-o", "-g", "-f"):
logger.writeError("Command-line option is not allowed in import mode: " + currArg)
return False
if currFlag == "-i":
if not validateOption(currFlag, currValue):
return False
self.interactive = True
elif currFlag == "-v":
if not validateOption(currFlag, currValue):
return False
self.verbose = True
else:
logger.writeError("File not found or command-line option not understood: " + currArg)
return False
if len(self.targetsToImport) == 0:
self.printUsage()
logger.writeError("No packages given when MixDown is in import mode")
return False
return True
def validate(self):
if not self.__validateOptionsDir(self.buildDir) or not\
self.__validateOptionsDir(self.downloadDir) or not\
self.__validateOptionsDir(self.logDir):
return False
if self.importMode and self.cleanMode:
logger.writeError(
"MixDown cannot be in both import mode and clean mode")
return False
if self.overrideFile == "" and len(self.overrideGroupNames) == 0:
logger.writeError(
"Override group name(s) given but no override file specified. Use command-line option '-o<filename>'."
)
return False
if self.overrideFile != "" and len(self.overrideGroupNames) == 0:
logger.writeError(
"Override file given but no override group(s) specified. Use command-line option '-g<Name>[,<Name>...]'."
)
return False
return True
def __validateDependsOnLists(self):
if len(self.targets) == 0:
return True
for currTarget in self.targets:
normalizedName = target.normalizeName(currTarget.name)
checkedDependencies = []
for dependency in currTarget.dependsOn:
normalizedDepedancy = target.normalizeName(dependency)
if normalizedDepedancy == normalizedName:
logger.writeError("Target cannot depend on itself", currTarget.name, "", self.path)
return False
if normalizedDepedancy in checkedDependencies:
logger.writeError("Target has duplicate dependency '" + dependency + "'", currTarget.name, "", self.path)
return False
if self.getTarget(dependency) is None:
logger.writeError("Target has non-existant dependency '" + dependency + "'", currTarget.name, "", self.path)
return False
checkedDependencies.append(normalizedDepedancy)
path = [target.normalizeName(self.targets[0].name)]
return self.__searchPathsForCycles(path)
def read(self):
if self.path == "":
logger.writeError("No project file was specified")
return False
f = open(self.path, "r")
try:
currTarget = None
lineCount = 0
for currLine in f:
lineCount += 1
lastPackageLineNumber = 0
currLine = str.strip(currLine)
if (currLine == "") or currLine.startswith('#') or currLine.startswith('//'):
pass
else:
colonIndex = currLine.find(":")
equalsIndex = currLine.find("=")
if (colonIndex == -1) and (equalsIndex == -1):
logger.writeError("Line could not be understood by project parser. Lines can be comments, defines, or target information.", "", "", self.path, lineCount)
return False
if ((colonIndex == -1) and (equalsIndex != -1)) or\
((colonIndex != -1) and (equalsIndex != -1) and (equalsIndex < colonIndex)):
currPair = currLine.split("=", 1)
currPair = currPair[0].strip(), currPair[1].strip()
currName = str.lower(currPair[0])
self.defines[currPair[0]] = currPair[1]
else:
currPair = currLine.split(":", 1)
currPair = currPair[0].strip(), currPair[1].strip()
currName = str.lower(currPair[0])
if (currName != "name") and (currTarget is None):
logger.writeError("'" + currPair[0] + "' declared before 'name' in Project file", "", "", self.path, lineCount)
return False
if currName == "name":
lastPackageLineNumber = lineCount
if currTarget != None:
if not self.__addTarget(currTarget, lastPackageLineNumber):
return False
currTarget = target.Target(currPair[1])
elif currName == "path":
if currTarget.path != "":
logger.writeError("Project targets can only have one 'Path' defined", "", "", self.path, lineCount)
return False
fullPath = os.path.abspath(currPair[1])
currTarget.origPath = fullPath
currTarget.path = fullPath
elif currName == "output":
if currTarget.outputPathSpecified:
logger.writeError("Project targets can only have one 'Output' defined", "", "", self.path, lineCount)
return False
currTarget.outputPath = currPair[1]
currTarget.outputPathSpecified = True
elif currName == "dependson":
if currTarget.dependsOn != []:
logger.writeError("Project targets can only have one 'DependsOn' defined (use a comma delimited list for multiple dependencies)", "", "", self.path, lineCount)
return False
if currPair[1] != "":
dependsOnList = utilityFunctions.stripItemsInList(currPair[1].split(","))
normalizedName = target.normalizeName(currTarget.name)
for dependency in dependsOnList:
if not re.match(r'[\w\.+_\-]+$', dependency):
logger.writeError("Dependancy name, '" + dependency + "', found with invalid character. Only alphanumeric is allowed.", currTarget.name, "", self.path, lineCount)
return False
if target.normalizeName(dependency) == normalizedName:
logger.writeError("Project targets cannot depend on themselves", currTarget.name, "", self.path, lineCount)
return False
currTarget.dependsOn = dependsOnList
elif currName == "aliases":
if currTarget.aliases != []:
logger.writeError("Project targets can only have one 'Aliases' defined (use a comma delimited list for multiple aliases)", "", "", self.path, lineCount)
return False
if currPair[1] != "":
aliases = utilityFunctions.stripItemsInList(currPair[1].split(","))
noralizedName = target.normalizeName(currTarget.name)
for alias in aliases:
if target.normalizeName(alias) == normalizedName:
logger.writeError("Project target alias cannot be same as its name", currTarget.name, "", self.path, lineCount)
return False
currTarget.aliases = aliases
elif currName == "skipsteps" or currName == "skipstep":
if currTarget.skipSteps != []:
logger.writeError("Project targets can only have one 'SkipSteps' defined (use a comma delimited list for multiple steps)", "", "", self.path, lineCount)
return False
currTarget.skipSteps = utilityFunctions.stripItemsInList(str.lower(currPair[1]).split(","))
elif currName == "prefix":
if currTarget.prefix != "":
logger.writeError("Project targets can only have one 'Prefix' defined", "", "", self.path, lineCount)
return False
currTarget.prefix = currPair[1]
elif currName == "version":
if currTarget.version != "":
logger.writeError("Project targets can only have one 'Version' defined", "", "", self.path, lineCount)
return False
currTarget.version = currPair[1]
elif currName in commands.buildSteps:
if currTarget.findBuildStep(currName) != None:
logger.writeError("Project targets can only have one '" + currName + "' defined", "", "", self.path, lineCount)
return False
currTarget.buildSteps.append(commands.BuildStep(currName, currPair[1]))
else:
logger.writeError("Cannot understand given line: '" + currLine + "'", "", "", self.path, lineCount)
return False
if not self.__addTarget(currTarget, lastPackageLineNumber):
return False
finally:
f.close()
return True
def processCommandline(self, commandline=[]):
if len(commandline) < 2:
self.printUsage()
return False
#Find mode
for arg in commandline[1:]: #skip script name
loweredArg = arg.lower()
if loweredArg == "--import":
self.importMode = True
elif loweredArg == "--clean":
self.cleanMode = True
self.cleanMixDown = False
elif loweredArg == "--profile":
self.profileMode = True
if (self.cleanMode and (self.profileMode or self.importMode)) or\
(self.profileMode and (self.importMode or self.cleanMode)):
logger.writeError(
"MixDown cannot be in both two command-line modes at the same time. Run 'MixDown --help' for instructions."
)
return False
if self.profileMode:
return self.__processProfileCommandline(commandline)
if self.importMode:
return self.__processImportCommandline(commandline)
for currArg in commandline[1:]: #skip script name
#Handle all options that don't follow -<letter><option>
if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"):
self.printUsage()
return False
elif os.path.splitext(currArg)[1] == ".md":
if not os.path.isfile(currArg):
logger.writeError("Project file " + currArg +
" does not exist")
return False
else:
self.projectFile = currArg
continue
elif currArg.lower() == "--clean":
continue
elif currArg.lower().startswith("--pythonpath="):
sys.path.append(os.path.abspath(currArg[13:]))
continue
#Handle all options that follow -<letter><option>
currFlag = currArg[:2].lower()
currValue = currArg[2:]
if currFlag in ("-i"):
logger.writeError(
"Command-line option is not allowed in build or clean mode: "
+ currArg)
return False
elif currFlag == "-p":
if not validateOptionPair(currFlag, currValue):
return False
defines.setPrefixDefines(self.defines,
os.path.abspath(currValue))
self.prefixDefined = True
elif currFlag == "-t":
if not validateOptionPair(currFlag, currValue):
return False
try:
count = int(currValue)
except ValueError:
count = 0
if count < 1:
logger.writeError("Positive numeric value needed " +
definePair)
return False
self.threadCount = count
elif currFlag == "-j":
if not validateOptionPair(currFlag, currValue):
return False
try:
count = int(currValue)
except ValueError:
count = 0
if count < 1:
logger.writeError("Positive numeric value needed " +
definePair)
return False
#Add "-j<jobSlots>" only if user defines -j on command-line
defines.setJobSlotsDefines(self.defines, currValue)
elif currFlag == "-l":
if not validateOptionPair(currFlag, currValue):
return False
self.logger = str.lower(currValue)
elif currFlag == "-r":
if not validateOption(currFlag, currValue):
return False
self.restart = True
elif currFlag == "-c":
if not validateOption(currFlag, currValue):
return False
self.continueBuilding = True
elif currFlag == "-b":
if not validateOptionPair(currFlag, currValue):
return False
self.buildDir = currValue
elif currFlag == "-w":
if not validateOptionPair(currFlag, currValue):
return False
self.downloadDir = currValue
elif currFlag == "-f":
if not validateOptionPair(currFlag, currValue):
return False
self.logDir = currValue
elif currFlag == "-k":
if not validateOption(currFlag, currValue):
return False
if self.cleanMode:
logger.writeError(
"Command-line arguments '--clean' and '-k' cannot both be used at the same time"
)
return False
self.cleanMixDown = False
elif currFlag == "-v":
if not validateOption(currFlag, currValue):
return False
self.verbose = True
elif currFlag == "-s":
if not validateOptionPair(currFlag, currValue):
return False
self.skipSteps = currValue
elif currFlag == "-o":
if not validateOptionPair(currFlag, currValue):
return False
self.overrideFile = currValue
elif currFlag == "-d":
if not validateOptionPair(currFlag, currValue):
return False
for definePair in currValue.split(","):
splitPair = definePair.split("=")
if len(splitPair) != 2:
logger.writeError("Invalid define pair given, " +
definePair)
return False
if splitPair[0].lower() in self.commandLineDefines:
logger.writeError("Define pair already given, " +
definePair)
return False
self.commandLineDefines[splitPair[0]] = splitPair[1]
elif currFlag == "-n":
if not validateOptionPair(currFlag, currValue):
return False
names = currValue.split(",")
if len(names) == 0:
logger.writeError(
"Build target names command-line option used but no names specified."
)
return False
for name in names:
currName = name.lower().strip()
if currName == "":
logger.writeError(
"Empty build target name given on command-line (option '-n')"
)
return False
self.targetsToBuild.append(currName)
elif currFlag == "-g":
if not validateOptionPair(currFlag, currValue):
return False
names = currValue.split(",")
if len(names) == 0:
logger.writeError(
"Override group command-line option used but no names specified."
)
return False
for name in names:
currName = name.lower().strip()
if currName == "":
logger.writeError(
"Empty override group name given on command-line (option '-g')"
)
return False
self.overrideGroupNames.append(currName)
else:
logger.writeError("Command-line argument '" + currArg +
"' not understood")
return False
return True
def validateOptionPair(flag, value):
if value == "":
logger.writeError(flag + " option requires a following value")
return False
return True
def validateOption(flag, value):
if value != "":
logger.writeError(flag + " option does not require a following value")
return False
return True
def readStatusLog(self, options):
lineCount = 0
if not os.path.exists(options.statusLogPath) or not os.path.isfile(options.statusLogPath):
return True, False
fd = open(options.statusLogPath, 'r')
try:
#path:<self.path>
l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount)
if EOF:
return False, True
if len(l) != 2:
logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount)
return False, True
if not self.__assureEqualsInStatusLog(l, "path", os.path.abspath(self.path)):
return False, True
#buildDir:<options.buildDir>
l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount)
if EOF:
return False, True
if len(l) != 2:
logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount)
return False, True
if not self.__assureEqualsInStatusLog(l, "buildDir", os.path.abspath(options.buildDir)):
return False, True
#prefix:<prefix>:<defined>
l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount)
if EOF:
return False, True
if len(l) != 3:
logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount)
return False, True
if not self.__assureEqualsInStatusLog(l, "prefix", os.path.abspath(options.defines[defines.mdPrefix[0]])):
return False, True
if l[2] != str(options.prefixDefined):
return False, True
#Target groups: one target line followed by multiple step lines
#<target.name>:<target.origPath>:<success>
#<step.name>:<step.command>:<step.restartPath>:<step.success>
while True:
l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount)
if EOF:
break
if len(l) == 3:
currTarget = self.getTarget(l[0])
if currTarget == None:
logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount)
continue
currTarget.origPath = l[1]
currTarget.success = utilityFunctions.boolToStr(l[2])
elif len(l) == 4:
currTarget = self.getTarget(l[0])
if currTarget == None:
continue
currStep = currTarget.findBuildStep(l[0])
if currStep == None:
continue
currStep.restartPath = l[2]
if currStep.restartPath == "":
currStep.restartPath = currTarget.origPath
currStep.success = utilityFunctions.boolToStr(l[3])
if currStep.command != l[1]:
currStep.success = False
else:
logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount)
return True, True
finally:
fd.close()
def validateOption(flag, value):
if value != "":
logger.writeError(flag + " option does not require a following value")
return False
return True
def readStatusLog(self, options):
lineCount = 0
if not os.path.exists(options.statusLogPath) or not os.path.isfile(
options.statusLogPath):
return True, False
fd = open(options.statusLogPath, 'r')
try:
#path:<self.path>
l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount)
if EOF:
return False, True
if len(l) != 2:
logger.writeError("Unknown line in status log:\n" +
':'.join(l),
filePath="options.statusLogPath",
lineNumber=lineCount)
return False, True
if not self.__assureEqualsInStatusLog(l, "path",
os.path.abspath(self.path)):
return False, True
#buildDir:<options.buildDir>
l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount)
if EOF:
return False, True
if len(l) != 2:
logger.writeError("Unknown line in status log:\n" +
':'.join(l),
filePath="options.statusLogPath",
lineNumber=lineCount)
return False, True
if not self.__assureEqualsInStatusLog(
l, "buildDir", os.path.abspath(options.buildDir)):
return False, True
#prefix:<prefix>:<defined>
l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount)
if EOF:
return False, True
if len(l) != 3:
logger.writeError("Unknown line in status log:\n" +
':'.join(l),
filePath="options.statusLogPath",
lineNumber=lineCount)
return False, True
if not self.__assureEqualsInStatusLog(
l, "prefix",
os.path.abspath(options.defines[defines.mdPrefix[0]])):
return False, True
if l[2] != str(options.prefixDefined):
return False, True
#Target groups: one target line followed by multiple step lines
#<target.name>:<target.origPath>:<success>
#<step.name>:<step.command>:<step.restartPath>:<step.success>
while True:
l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount)
if EOF:
break
if len(l) == 3:
currTarget = self.getTarget(l[0])
if currTarget == None:
logger.writeError("Unknown line in status log:\n" +
':'.join(l),
filePath="options.statusLogPath",
lineNumber=lineCount)
continue
currTarget.origPath = l[1]
currTarget.success = utilityFunctions.boolToStr(l[2])
elif len(l) == 4:
currTarget = self.getTarget(l[0])
if currTarget == None:
continue
currStep = currTarget.findBuildStep(l[0])
if currStep == None:
continue
currStep.restartPath = l[2]
if currStep.restartPath == "":
currStep.restartPath = currTarget.origPath
currStep.success = utilityFunctions.boolToStr(l[3])
if currStep.command != l[1]:
currStep.success = False
else:
logger.writeError("Unknown line in status log:\n" +
':'.join(l),
filePath="options.statusLogPath",
lineNumber=lineCount)
return True, True
finally:
fd.close()
def read(self):
if self.path == "":
logger.writeError("No project file was specified")
return False
f = open(self.path, "r")
try:
currTarget = None
lineCount = 0
for currLine in f:
lineCount += 1
lastPackageLineNumber = 0
currLine = str.strip(currLine)
if (currLine == ""
) or currLine.startswith('#') or currLine.startswith('//'):
pass
else:
colonIndex = currLine.find(":")
equalsIndex = currLine.find("=")
if (colonIndex == -1) and (equalsIndex == -1):
logger.writeError(
"Line could not be understood by project parser. Lines can be comments, defines, or target information.",
"", "", self.path, lineCount)
return False
if ((colonIndex == -1) and (equalsIndex != -1)) or\
((colonIndex != -1) and (equalsIndex != -1) and (equalsIndex < colonIndex)):
currPair = currLine.split("=", 1)
currPair = currPair[0].strip(), currPair[1].strip()
currName = str.lower(currPair[0])
self.defines[currPair[0]] = currPair[1]
else:
currPair = currLine.split(":", 1)
currPair = currPair[0].strip(), currPair[1].strip()
currName = str.lower(currPair[0])
if (currName != "name") and (currTarget is None):
logger.writeError(
"'" + currPair[0] +
"' declared before 'name' in Project file", "",
"", self.path, lineCount)
return False
if currName == "name":
lastPackageLineNumber = lineCount
if currTarget != None:
if not self.__addTarget(
currTarget, lastPackageLineNumber):
return False
currTarget = target.Target(currPair[1])
elif currName == "path":
if currTarget.path != "":
logger.writeError(
"Project targets can only have one 'Path' defined",
"", "", self.path, lineCount)
return False
fullPath = os.path.abspath(currPair[1])
currTarget.origPath = fullPath
currTarget.path = fullPath
elif currName == "output":
if currTarget.outputPathSpecified:
logger.writeError(
"Project targets can only have one 'Output' defined",
"", "", self.path, lineCount)
return False
currTarget.outputPath = currPair[1]
currTarget.outputPathSpecified = True
elif currName == "dependson":
if currTarget.dependsOn != []:
logger.writeError(
"Project targets can only have one 'DependsOn' defined (use a comma delimited list for multiple dependencies)",
"", "", self.path, lineCount)
return False
if currPair[1] != "":
dependsOnList = utilityFunctions.stripItemsInList(
currPair[1].split(","))
normalizedName = target.normalizeName(
currTarget.name)
for dependency in dependsOnList:
if not re.match(r'[\w\.+_\-]+$',
dependency):
logger.writeError(
"Dependancy name, '" + dependency +
"', found with invalid character. Only alphanumeric is allowed.",
currTarget.name, "", self.path,
lineCount)
return False
if target.normalizeName(
dependency) == normalizedName:
logger.writeError(
"Project targets cannot depend on themselves",
currTarget.name, "", self.path,
lineCount)
return False
currTarget.dependsOn = dependsOnList
elif currName == "aliases":
if currTarget.aliases != []:
logger.writeError(
"Project targets can only have one 'Aliases' defined (use a comma delimited list for multiple aliases)",
"", "", self.path, lineCount)
return False
if currPair[1] != "":
aliases = utilityFunctions.stripItemsInList(
currPair[1].split(","))
noralizedName = target.normalizeName(
currTarget.name)
for alias in aliases:
if target.normalizeName(
alias) == normalizedName:
logger.writeError(
"Project target alias cannot be same as its name",
currTarget.name, "", self.path,
lineCount)
return False
currTarget.aliases = aliases
elif currName == "skipsteps" or currName == "skipstep":
if currTarget.skipSteps != []:
logger.writeError(
"Project targets can only have one 'SkipSteps' defined (use a comma delimited list for multiple steps)",
"", "", self.path, lineCount)
return False
currTarget.skipSteps = utilityFunctions.stripItemsInList(
str.lower(currPair[1]).split(","))
elif currName == "prefix":
if currTarget.prefix != "":
logger.writeError(
"Project targets can only have one 'Prefix' defined",
"", "", self.path, lineCount)
return False
currTarget.prefix = currPair[1]
elif currName == "version":
if currTarget.version != "":
logger.writeError(
"Project targets can only have one 'Version' defined",
"", "", self.path, lineCount)
return False
currTarget.version = currPair[1]
elif currName in commands.buildSteps:
if currTarget.findBuildStep(currName) != None:
logger.writeError(
"Project targets can only have one '" +
currName + "' defined", "", "", self.path,
lineCount)
return False
currTarget.buildSteps.append(
commands.BuildStep(currName, currPair[1]))
else:
logger.writeError(
"Cannot understand given line: '" + currLine +
"'", "", "", self.path, lineCount)
return False
if not self.__addTarget(currTarget, lastPackageLineNumber):
return False
finally:
f.close()
return True
def importTargets(options):
logger.setLogger("console")
finalTargets = []
ignoredTargets = []
partialImport = False
fetchStep = commands.BuildStep("fetch", commands.__getFetchCommand(None))
unpackStep = commands.BuildStep("unpack", commands.__getUnpackCommand(None))
tempDir = tempfile.mkdtemp(prefix="mixdown-")
options.downloadDir = os.path.join(tempDir, "mdDownloads")
while len(options.targetsToImport) != 0:
target = options.targetsToImport.pop(0)
logger.writeMessage("Analyzing target...", target.name)
logger.writeMessage("Extracting target...", target.name)
target.outputPath = os.path.join(tempDir, target.name)
target.currBuildStep = fetchStep
if not commands.buildStepActor(target, options, None):
utilityFunctions.removeDir(tempDir)
return None, False
target.currBuildStep = unpackStep
if not commands.buildStepActor(target, options, None):
utilityFunctions.removeDir(tempDir)
return None, False
#Generate build files and find possible dependencies
possibleDeps = []
if cmake.isCMakeProject(target.path):
logger.writeMessage("CMake project found...", target.name)
logger.writeMessage("Analyzing for dependencies...", target.name)
possibleDeps = cmake.getDependencies(target.path, target.name)
elif autoTools.isAutoToolsProject(target.path):
logger.writeMessage("Auto Tools project found...", target.name)
if not os.path.exists(os.path.join(target.path, "configure")):
if not autoTools.generateConfigureFiles(target.path, target.name):
utilityFunctions.removeDir(tempDir)
return None, False
logger.writeMessage("Analyzing for dependencies...", target.name)
possibleDeps = autoTools.getDependencies(target.path, target.name)
if possibleDeps == None:
target.comment = "Unable to parse 'configure --help' output. MixDown cannot determine dependencies for this target."
logger.writeError(target.comment, target.name)
partialImport = True
possibleDeps = []
elif make.isMakeProject(target.path):
target.comment = "Make project found. MixDown cannot determine dependencies from Make projects."
logger.writeError(target.comment, target.name)
partialImport = True
else:
target.comment = "Unknown build system found. MixDown cannot determine dependencies or build commands."
logger.writeError(target.comment, target.name)
partialImport = True
#Find actual dependencies
for possibleDependency in possibleDeps:
if getTarget(possibleDependency, finalTargets + options.targetsToImport):
logger.writeMessage("Known dependency found (" + possibleDependency + ")", target.name)
target.dependsOn.append(possibleDependency)
continue
elif options.interactive and possibleDependency in ignoredTargets:
logger.writeMessage("Previously ignored dependency found (" + possibleDependency + ")", target.name)
continue
if searchForPossibleAliasInList(possibleDependency, finalTargets + options.targetsToImport, options.interactive):
target.dependsOn.append(possibleDependency)
elif not options.interactive:
logger.writeMessage("Ignoring unknown dependency (" + possibleDependency + ")", target.name)
else:
logger.writeMessage("Unknown dependency found (" + possibleDependency + ")", target.name)
userInput = raw_input(possibleDependency + ": Input location, target name, or blank to ignore:").strip()
if userInput == "":
ignoredTargets.append(possibleDependency)
elif os.path.isfile(userInput) or os.path.isdir(userInput) or utilityFunctions.isURL(userInput):
name = target.targetPathToName(userInput)
if name == "":
return None, False
newTarget = target.Target(name, userInput)
options.targetsToImport.append(newTarget)
if target.normalizeName(possibleDependency) != target.normalizeName(userInput):
newTarget.aliases.append(possibleDependency)
target.dependsOn.append(possibleDependency)
else:
aliasTarget = getTarget(userInput, finalTargets + options.targetsToImport, possibleDependency)
if aliasTarget != None:
logger.writeMessage("Alias added (" + userInput + ")", aliasTarget.name)
target.dependsOn.append(possibleDependency)
else:
aliasLocation = raw_input(userInput + ": Target name not found in any known targets. Location of new target:").strip()
if os.path.isfile(aliasLocation) or os.path.isdir(aliasLocation) or utilityFunctions.isURL(aliasLocation):
name = target.targetPathToName(aliasLocation)
if name == "":
return None, False
newTarget = target.Target(name, aliasLocation)
notReviewedTargets.append(newTarget)
if target.normalizeName(possibleDependency) != target.normalizeName(aliasLocation):
newTarget.aliases.append(possibleDependency)
target.dependsOn.append(possibleDependency)
else:
logger.writeError(userInput + ": Alias location not understood.", exitProgram=True)
finalTargets.append(target)
#Create project for targets
projects = project.Project("ProjectNameNotDetermined", finalTargets)
if not projects.examine(options):
logger.writeError("Project failed examination", exitProgram=True)
if not projects.validate(options):
logger.writeError("Project failed validation", exitProgram=True)
mainTargetPath = projects.targets[0].origPath
if utilityFunctions.isURL(mainTargetPath):
mainTargetPath = utilityFunctions.URLToFilename(mainTargetPath)
mainTargetName, mainTargetVersion = utilityFunctions.splitFileName(mainTargetPath)
if mainTargetVersion != "":
projects.name = mainTargetName + "-" + mainTargetVersion
else:
projects.name = mainTargetName
projects.path = projects.name + ".md"
for target in projects.targets:
target.outputPath = ""
if projects.examine(options):
logger.writeMessage("\nFinal targets...\n\n" + str(projects))
projects.write()
utilityFunctions.removeDir(tempDir)
return projects, partialImport
def __validateOptionsDir(self, path):
if os.path.isfile(self.buildDir):
logger.writeError("Cannot create directory by MixDown, a file by the same name already exists: " + path)
return False
return True
def readGroups(filename):
if not os.path.exists(filename):
logger.writeError(
"Given Override file path did not exist. Check your -o command-line option.",
filePath=filename)
return None
groups = list()
overrideGroup = None
tokenizer = Tokenizer(filename)
if not tokenizer.tokenize():
return None
tokens = tokenizer.tokens
i = 0
lengthOfTokens = len(tokens)
while i < lengthOfTokens:
#Syntax start: <name>[, <name>]*
overrideGroup = OverrideGroup()
while not (tokens[i].type == TokenType.Symbol
and tokens[i].value == '{'):
if len(overrideGroup.name) > 50:
logger.writeError("Override group name exceeded limit (50).",
filePath=filename)
return None
if tokens[i].type == TokenType.Identifier:
overrideGroup.name.append(tokens[i].value.lower())
i += 1
if i >= lengthOfTokens:
logger.writeError(
"Parsing ended inside of Override group. Please finish file and re-run MixDown.",
filePath=filename)
return None
if tokens[i].type == TokenType.Symbol and tokens[
i].value == '{':
#End of override group name syntax hit
break
elif tokens[i].type == TokenType.Symbol and tokens[
i].value == ',':
i += 1
if i >= lengthOfTokens:
logger.writeError(
"Parsing ended inside of Override group. Please finish file and re-run MixDown.",
filePath=filename)
return None
else:
logger.writeError("Expected ',' got '" + tokens[i].value +
"'",
filePath=filename)
return None
else:
logger.writeError("Expected Override group name, got '" +
tokens[i].value + "'",
filePath=filename)
return None
#Syntax end
#Syntax start:
#{
# <Identifier> = <string>
# ...
# <Identifier> = <string>
#}
if not (tokens[i].type == TokenType.Symbol and tokens[i].value == '{'):
logger.writeError("Expected '{' got '" + tokens[i].value + "'",
filePath=filename)
return None
i += 1
if i >= lengthOfTokens:
logger.writeError(
"Parsing ended inside of Override group. Please finish file and re-run MixDown.",
filePath=filename)
return None
while True:
if tokens[i].type == TokenType.Symbol and tokens[i].value == '}':
i += 1
break
overrideName = ""
overrideValue = ""
#Syntax start: <Identifier> = <string>
if not tokens[i].type == TokenType.Identifier:
logger.writeError("Expected Override identifier got '" +
tokens[i].value + "'",
filePath=filename)
return None
overrideNameOriginal = tokens[i].value
overrideName = tokens[i].value.lower()
i += 1
if i >= lengthOfTokens:
logger.writeError(
"Parsing ended inside of Override group. Please finish file and re-run MixDown.",
filePath=filename)
return None
if not (tokens[i].type == TokenType.Symbol
and tokens[i].value == '='):
logger.writeError("Expected '=' got '" + tokens[i].value + "'",
filePath=filename)
return None
i += 1
if i >= lengthOfTokens:
logger.writeError(
"Parsing ended inside of Override group. Please finish file and re-run MixDown.",
filePath=filename)
return None
if not tokens[i].type == TokenType.String:
logger.writeError("Expected Override value string got '" +
tokens[i].value + "'",
filePath=filename)
return None
overrideString = tokens[i].value
i += 1
if i >= lengthOfTokens:
logger.writeError(
"Parsing ended inside of Override group. Please finish file and re-run MixDown.",
filePath=filename)
return None
overrideGroup[overrideName] = overrideString
#Syntax end
#Syntax end
overrideGroupNames = ", ".join(overrideGroup)
for group in groups:
if ", ".join(group.name) == overrideGroupNames:
logger.writeError("Duplicate override group name found: " +
overrideGroupNames,
filePath=options.overrideFile)
return None
groups.append(overrideGroup)
return groups
def processCommandline(self, commandline=[]):
if len(commandline) < 2:
self.printUsage()
return False
#Find mode
for arg in commandline[1:]: #skip script name
loweredArg = arg.lower()
if loweredArg == "--import":
self.importMode = True
elif loweredArg == "--clean":
self.cleanMode = True
self.cleanMixDown = False
elif loweredArg == "--profile":
self.profileMode = True
if (self.cleanMode and (self.profileMode or self.importMode)) or\
(self.profileMode and (self.importMode or self.cleanMode)):
logger.writeError("MixDown cannot be in both two command-line modes at the same time. Run 'MixDown --help' for instructions.")
return False
if self.profileMode:
return self.__processProfileCommandline(commandline)
if self.importMode:
return self.__processImportCommandline(commandline)
for currArg in commandline[1:]: #skip script name
#Handle all options that don't follow -<letter><option>
if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"):
self.printUsage()
return False
elif os.path.splitext(currArg)[1] == ".md":
if not os.path.isfile(currArg):
logger.writeError("Project file " + currArg + " does not exist")
return False
else:
self.projectFile = currArg
continue
elif currArg.lower() == "--clean":
continue
elif currArg.lower().startswith("--pythonpath="):
sys.path.append(os.path.abspath(currArg[13:]))
continue
#Handle all options that follow -<letter><option>
currFlag = currArg[:2].lower()
currValue = currArg[2:]
if currFlag in ("-i"):
logger.writeError("Command-line option is not allowed in build or clean mode: " + currArg)
return False
elif currFlag == "-p":
if not validateOptionPair(currFlag, currValue):
return False
defines.setPrefixDefines(self.defines, os.path.abspath(currValue))
self.prefixDefined = True
elif currFlag == "-t":
if not validateOptionPair(currFlag, currValue):
return False
try:
count = int(currValue)
except ValueError:
count = 0
if count < 1:
logger.writeError("Positive numeric value needed " + definePair)
return False
self.threadCount = count
elif currFlag == "-j":
if not validateOptionPair(currFlag, currValue):
return False
try:
count = int(currValue)
except ValueError:
count = 0
if count < 1:
logger.writeError("Positive numeric value needed " + definePair)
return False
#Add "-j<jobSlots>" only if user defines -j on command-line
defines.setJobSlotsDefines(self.defines, currValue)
elif currFlag == "-l":
if not validateOptionPair(currFlag, currValue):
return False
self.logger = str.lower(currValue)
elif currFlag == "-r":
if not validateOption(currFlag, currValue):
return False
self.restart = True
elif currFlag == "-c":
if not validateOption(currFlag, currValue):
return False
self.continueBuilding = True
elif currFlag == "-b":
if not validateOptionPair(currFlag, currValue):
return False
self.buildDir = currValue
elif currFlag == "-w":
if not validateOptionPair(currFlag, currValue):
return False
self.downloadDir = currValue
elif currFlag == "-f":
if not validateOptionPair(currFlag, currValue):
return False
self.logDir = currValue
elif currFlag == "-k":
if not validateOption(currFlag, currValue):
return False
if self.cleanMode:
logger.writeError("Command-line arguments '--clean' and '-k' cannot both be used at the same time")
return False
self.cleanMixDown = False
elif currFlag == "-v":
if not validateOption(currFlag, currValue):
return False
self.verbose = True
elif currFlag == "-s":
if not validateOptionPair(currFlag, currValue):
return False
self.skipSteps = currValue
elif currFlag == "-o":
if not validateOptionPair(currFlag, currValue):
return False
self.overrideFile = currValue
elif currFlag == "-d":
if not validateOptionPair(currFlag, currValue):
return False
for definePair in currValue.split(","):
splitPair = definePair.split("=")
if len(splitPair) != 2:
logger.writeError("Invalid define pair given, " + definePair)
return False
if splitPair[0].lower() in self.commandLineDefines:
logger.writeError("Define pair already given, " + definePair)
return False
self.commandLineDefines[splitPair[0]] = splitPair[1]
elif currFlag == "-n":
if not validateOptionPair(currFlag, currValue):
return False
names = currValue.split(",")
if len(names) == 0:
logger.writeError("Build target names command-line option used but no names specified.")
return False
for name in names:
currName = name.lower().strip()
if currName == "":
logger.writeError("Empty build target name given on command-line (option '-n')")
return False
self.targetsToBuild.append(currName)
elif currFlag == "-g":
if not validateOptionPair(currFlag, currValue):
return False
names = currValue.split(",")
if len(names) == 0:
logger.writeError("Override group command-line option used but no names specified.")
return False
for name in names:
currName = name.lower().strip()
if currName == "":
logger.writeError("Empty override group name given on command-line (option '-g')")
return False
self.overrideGroupNames.append(currName)
else:
logger.writeError("Command-line argument '" + currArg + "' not understood")
return False
return True
def importTargets(options):
logger.setLogger("console")
finalTargets = []
ignoredTargets = []
partialImport = False
fetchStep = commands.BuildStep("fetch", commands.__getFetchCommand(None))
unpackStep = commands.BuildStep("unpack",
commands.__getUnpackCommand(None))
tempDir = tempfile.mkdtemp(prefix="mixdown-")
options.downloadDir = os.path.join(tempDir, "mdDownloads")
while len(options.targetsToImport) != 0:
target = options.targetsToImport.pop(0)
logger.writeMessage("Analyzing target...", target.name)
logger.writeMessage("Extracting target...", target.name)
target.outputPath = os.path.join(tempDir, target.name)
target.currBuildStep = fetchStep
if not commands.buildStepActor(target, options, None):
utilityFunctions.removeDir(tempDir)
return None, False
target.currBuildStep = unpackStep
if not commands.buildStepActor(target, options, None):
utilityFunctions.removeDir(tempDir)
return None, False
#Generate build files and find possible dependencies
possibleDeps = []
if cmake.isCMakeProject(target.path):
logger.writeMessage("CMake project found...", target.name)
logger.writeMessage("Analyzing for dependencies...", target.name)
possibleDeps = cmake.getDependencies(target.path, target.name)
elif autoTools.isAutoToolsProject(target.path):
logger.writeMessage("Auto Tools project found...", target.name)
if not os.path.exists(os.path.join(target.path, "configure")):
if not autoTools.generateConfigureFiles(
target.path, target.name):
utilityFunctions.removeDir(tempDir)
return None, False
logger.writeMessage("Analyzing for dependencies...", target.name)
possibleDeps = autoTools.getDependencies(target.path, target.name)
if possibleDeps == None:
target.comment = "Unable to parse 'configure --help' output. MixDown cannot determine dependencies for this target."
logger.writeError(target.comment, target.name)
partialImport = True
possibleDeps = []
elif make.isMakeProject(target.path):
target.comment = "Make project found. MixDown cannot determine dependencies from Make projects."
logger.writeError(target.comment, target.name)
partialImport = True
else:
target.comment = "Unknown build system found. MixDown cannot determine dependencies or build commands."
logger.writeError(target.comment, target.name)
partialImport = True
#Find actual dependencies
for possibleDependency in possibleDeps:
if getTarget(possibleDependency,
finalTargets + options.targetsToImport):
logger.writeMessage(
"Known dependency found (" + possibleDependency + ")",
target.name)
target.dependsOn.append(possibleDependency)
continue
elif options.interactive and possibleDependency in ignoredTargets:
logger.writeMessage(
"Previously ignored dependency found (" +
possibleDependency + ")", target.name)
continue
if searchForPossibleAliasInList(
possibleDependency, finalTargets + options.targetsToImport,
options.interactive):
target.dependsOn.append(possibleDependency)
elif not options.interactive:
logger.writeMessage(
"Ignoring unknown dependency (" + possibleDependency + ")",
target.name)
else:
logger.writeMessage(
"Unknown dependency found (" + possibleDependency + ")",
target.name)
userInput = raw_input(
possibleDependency +
": Input location, target name, or blank to ignore:"
).strip()
if userInput == "":
ignoredTargets.append(possibleDependency)
elif os.path.isfile(userInput) or os.path.isdir(
userInput) or utilityFunctions.isURL(userInput):
name = target.targetPathToName(userInput)
if name == "":
return None, False
newTarget = target.Target(name, userInput)
options.targetsToImport.append(newTarget)
if target.normalizeName(
possibleDependency) != target.normalizeName(
userInput):
newTarget.aliases.append(possibleDependency)
target.dependsOn.append(possibleDependency)
else:
aliasTarget = getTarget(
userInput, finalTargets + options.targetsToImport,
possibleDependency)
if aliasTarget != None:
logger.writeMessage("Alias added (" + userInput + ")",
aliasTarget.name)
target.dependsOn.append(possibleDependency)
else:
aliasLocation = raw_input(
userInput +
": Target name not found in any known targets. Location of new target:"
).strip()
if os.path.isfile(aliasLocation) or os.path.isdir(
aliasLocation) or utilityFunctions.isURL(
aliasLocation):
name = target.targetPathToName(aliasLocation)
if name == "":
return None, False
newTarget = target.Target(name, aliasLocation)
notReviewedTargets.append(newTarget)
if target.normalizeName(possibleDependency
) != target.normalizeName(
aliasLocation):
newTarget.aliases.append(possibleDependency)
target.dependsOn.append(possibleDependency)
else:
logger.writeError(
userInput + ": Alias location not understood.",
exitProgram=True)
finalTargets.append(target)
#Create project for targets
projects = project.Project("ProjectNameNotDetermined", finalTargets)
if not projects.examine(options):
logger.writeError("Project failed examination", exitProgram=True)
if not projects.validate(options):
logger.writeError("Project failed validation", exitProgram=True)
mainTargetPath = projects.targets[0].origPath
if utilityFunctions.isURL(mainTargetPath):
mainTargetPath = utilityFunctions.URLToFilename(mainTargetPath)
mainTargetName, mainTargetVersion = utilityFunctions.splitFileName(
mainTargetPath)
if mainTargetVersion != "":
projects.name = mainTargetName + "-" + mainTargetVersion
else:
projects.name = mainTargetName
projects.path = projects.name + ".md"
for target in projects.targets:
target.outputPath = ""
if projects.examine(options):
logger.writeMessage("\nFinal targets...\n\n" + str(projects))
projects.write()
utilityFunctions.removeDir(tempDir)
return projects, partialImport
def validateOptionPair(flag, value):
if value == "":
logger.writeError(flag + " option requires a following value")
return False
return True
def buildStepActor(target, options, project, lock=None):
if target.isStepToBeSkipped(target.currBuildStep.name):
try:
if lock:
lock.acquire()
logger.reportSkipped(target.name, target.currBuildStep.name, "Target specified to skip step")
finally:
if lock:
lock.release()
return True
if target.isStepPreviouslyDone(target.currBuildStep.name):
try:
if lock:
lock.acquire()
logger.reportSkipped(
target.name, target.currBuildStep.name, "Build step successfully built in previous MixDown build"
)
finally:
if lock:
lock.release()
return True
try:
if lock:
lock.acquire()
logger.reportStart(target.name, target.currBuildStep.name)
# Refresh defines before start of every step
project.setTargetFieldsAsDefines(options.defines)
finally:
if lock:
lock.release()
returnCode = None
timeStart = time.time()
command = options.defines.expand(target.currBuildStep.command)
isPythonCommand, namespace, function = python.parsePythonCommand(command)
if isPythonCommand:
success = python.callPythonCommand(namespace, function, target, options, project)
if not success:
returnCode = 1
else:
returnCode = 0
else:
try:
if lock:
lock.acquire()
logger.writeMessage("Executing command: " + command, target.name, target.currBuildStep.name, True)
finally:
if lock:
lock.release()
if not os.path.exists(target.path):
logger.writeError(
target.name
+ "'s path does not exist when about to execute build command in step "
+ target.currBuildStep.name
+ ".",
filePath=target.path,
)
returnCode = 1
else:
outFd = logger.getOutFd(target.name, target.currBuildStep.name)
returnCode = utilityFunctions.executeSubProcess(command, target.path, outFd)
timeFinished = time.time()
timeElapsed = timeFinished - timeStart
if returnCode != 0:
target.currBuildStep.success = False
try:
if lock:
lock.acquire()
logger.reportFailure(target.name, target.currBuildStep.name, timeElapsed, returnCode)
finally:
if lock:
lock.release()
return False
target.currBuildStep.success = True
try:
if lock:
lock.acquire()
logger.reportSuccess(target.name, target.currBuildStep.name, timeElapsed)
finally:
if lock:
lock.release()
return True
def readGroups(filename):
if not os.path.exists(filename):
logger.writeError("Given Override file path did not exist. Check your -o command-line option.", filePath=filename)
return None
groups = list()
overrideGroup = None
tokenizer = Tokenizer(filename)
if not tokenizer.tokenize():
return None
tokens = tokenizer.tokens
i = 0
lengthOfTokens = len(tokens)
while i < lengthOfTokens:
#Syntax start: <name>[, <name>]*
overrideGroup = OverrideGroup()
while not (tokens[i].type == TokenType.Symbol and tokens[i].value == '{'):
if len(overrideGroup.name) > 50:
logger.writeError("Override group name exceeded limit (50).", filePath=filename)
return None
if tokens[i].type == TokenType.Identifier:
overrideGroup.name.append(tokens[i].value.lower())
i += 1
if i >= lengthOfTokens:
logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename)
return None
if tokens[i].type == TokenType.Symbol and tokens[i].value == '{':
#End of override group name syntax hit
break
elif tokens[i].type == TokenType.Symbol and tokens[i].value == ',':
i += 1
if i >= lengthOfTokens:
logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename)
return None
else:
logger.writeError("Expected ',' got '" + tokens[i].value + "'", filePath=filename)
return None
else:
logger.writeError("Expected Override group name, got '" + tokens[i].value + "'", filePath=filename)
return None
#Syntax end
#Syntax start:
#{
# <Identifier> = <string>
# ...
# <Identifier> = <string>
#}
if not (tokens[i].type == TokenType.Symbol and tokens[i].value == '{'):
logger.writeError("Expected '{' got '" + tokens[i].value + "'", filePath=filename)
return None
i += 1
if i >= lengthOfTokens:
logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename)
return None
while True:
if tokens[i].type == TokenType.Symbol and tokens[i].value == '}':
i += 1
break
overrideName = ""
overrideValue = ""
#Syntax start: <Identifier> = <string>
if not tokens[i].type == TokenType.Identifier:
logger.writeError("Expected Override identifier got '" + tokens[i].value + "'", filePath=filename)
return None
overrideNameOriginal = tokens[i].value
overrideName = tokens[i].value.lower()
i += 1
if i >= lengthOfTokens:
logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename)
return None
if not (tokens[i].type == TokenType.Symbol and tokens[i].value == '='):
logger.writeError("Expected '=' got '" + tokens[i].value + "'", filePath=filename)
return None
i += 1
if i >= lengthOfTokens:
logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename)
return None
if not tokens[i].type == TokenType.String:
logger.writeError("Expected Override value string got '" + tokens[i].value + "'", filePath=filename)
return None
overrideString = tokens[i].value
i += 1
if i >= lengthOfTokens:
logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename)
return None
overrideGroup[overrideName] = overrideString
#Syntax end
#Syntax end
overrideGroupNames = ", ".join(overrideGroup)
for group in groups:
if ", ".join(group.name) == overrideGroupNames:
logger.writeError("Duplicate override group name found: " + overrideGroupNames, filePath=options.overrideFile)
return None
groups.append(overrideGroup)
return groups