def validate(self, mdOptions): normalizedName = normalizeName(self.name) if normalizedName == "": return False if self.path == "": return False for alias in self.aliases: if normalizeName(alias) == normalizedName: logger.writeError("Target's alias cannot be same as it's name", self.name) return False #Check for write access to install directories used in commands. if mdOptions.targetSpecifiedToBuild(self.name) and not mdOptions.cleanMode and not mdOptions.importMode: for buildStep in self.buildSteps: if buildStep.command == "": continue expandedCommand = mdOptions.defines.expand(buildStep.command) installDir = autoTools.getInstallDir(expandedCommand) if installDir == "": installDir = cmake.getInstallDir(expandedCommand) if installDir != "" and not utilityFunctions.haveWriteAccess(installDir): logger.writeError("No write access to used install directory: " + installDir, self.name, buildStep.name, mdOptions.projectFile) if not mdOptions.prefixDefined: logger.writeMessage("Use command-line option '-p<install path>' or running MixDown with superuser privileges (sudo)") else: logger.writeMessage("Choose a different install directory for command-line option '-p<install path>'") return False return True
def _getToken(self, line, previousToken): token = Token() start = 0 finish = 0 lineLength = len(line) if previousToken.type == TokenType.Symbol and previousToken.value == "=": token.type = TokenType.String token.value = line line = "" elif self._isValidIdentifierCharacter(line[0]): token.type = TokenType.Identifier finish = 1 while finish < lineLength: if not self._isValidIdentifierCharacter(line[finish]): break finish += 1 token.value = line[0:finish] line = line[finish:] elif line[0] in symbols: token.type = TokenType.Symbol token.value = line[0] line = line[1:] else: logger.writeError("Tokenizer failed on: " + line, filePath=self.fileName) return None, line line = line.lstrip() return token, line
def targetPathToName(path): name = "" path = path.strip() if git.isGitRepo(path): if os.path.isdir(path): name = os.path.basename(path) elif utilityFunctions.isURL(path): name = utilityFunctions.URLToFilename(path) if name.endswith(".git"): name = name[:-4] elif svn.isSvnRepo(path): if path.endswith(os.sep): path = path[:-1] if path.endswith("/trunk") or path.endswith("\trunk"): path = path[:-6] if os.path.isdir(path): name = os.path.basename(path) elif utilityFunctions.isURL(path): name = utilityFunctions.URLToFilename(path) elif utilityFunctions.isURL(path): name = utilityFunctions.URLToFilename(path) name = utilityFunctions.splitFileName(name)[0] elif os.path.isfile(path) and utilityFunctions.validateCompressedFile(path, logger): name = utilityFunctions.splitFileName(path)[0] elif os.path.isdir(path): if path.endswith(os.sep): name = os.path.basename(path[:-1]) else: name = os.path.basename(path) else: logger.writeError("Could not convert given target path to name: " + path) return name
def __validateOptionsDir(self, path): if os.path.isfile(self.buildDir): logger.writeError( "Cannot create directory by MixDown, a file by the same name already exists: " + path) return False return True
def __assureEqualsInStatusLog(self, l, name, value): #Note just get rid of this function..... if l[0].lower() != name.lower(): logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath") return False if l[1] != value: return False return True
def __addTarget(self, targets, lineCount=0): if targets.name == "" or targets.path == "": logger.writeError("New target started before previous was finished, all targets require atleast 'Name' and 'Path' to be declared", "", "", self.path, lineCount) return False for currTarget in self.targets: if target.normalizeName(targets.name) == target.normalizeName(currTarget.name): logger.writeError("Cannot have more than one project target by the same name", currTarget.name, "", self.path, lineCount) return False self.targets.append(targets) return True
def __assignDepthToTargetList(self): q = Queue.Queue() for targets in self.targets: q.put(targets.name) while not q.empty(): currName = q.get() currTarget = self.getTarget(currName) for currChildName in currTarget.dependsOn: currChildTarget = self.getTarget(currChildName) if not currChildTarget: logger.writeError("Target '" + currTarget.name + "' has non-existant dependency '" + currChildName + "'", "", self.path, exitProgram=True) if currChildTarget.dependencyDepth < (currTarget.dependencyDepth + 1): currChildTarget.dependencyDepth = currTarget.dependencyDepth + 1 q.put(currChildName)
def determineOutputPath(self, mdOptions): if self.outputPathSpecified and self.outputPath != "": return self.outputPath else: targetsBuildDir = os.path.join(mdOptions.buildDir, self.name) if mdOptions.cleanMode: if os.path.exists(targetsBuildDir) and os.path.isdir(targetsBuildDir): return targetsBuildDir elif os.path.isdir(self.path): return self.path else: logger.writeError("Output path could not be located, define in project file with \"output=<path>\"", self.name, "clean") return "" return targetsBuildDir
def examine(self, options): if self.__examined: return True else: if len(self.targets) < 1: logger.writeError("Project has no targets") return False self.__assignDepthToTargetList() self.targets = self.__sortTargetList(self.targets) self.__expandDependancyLists() for targets in self.targets: if not targets.examine(options): return False self.__examined = True return True
def __addTarget(self, targets, lineCount=0): if targets.name == "" or targets.path == "": logger.writeError( "New target started before previous was finished, all targets require atleast 'Name' and 'Path' to be declared", "", "", self.path, lineCount) return False for currTarget in self.targets: if target.normalizeName(targets.name) == target.normalizeName( currTarget.name): logger.writeError( "Cannot have more than one project target by the same name", currTarget.name, "", self.path, lineCount) return False self.targets.append(targets) return True
def addSkipStepFromOptions(self, options): if options.skipSteps == '' or options.skipSteps == None: return True skipSteps = options.skipSteps for item in skipSteps.split(","): pair = item.split(":") if len(pair) != 2 or pair[0] == "" or pair[1] == "": logger.writeError("Invalid command-line -s pair found: " + item) logger.writeMessage("Proper use: -s[Semi-colon delimited list of Skip Step pairs]") logger.writeMessage("Skip Step Pair: [targetName]:[Steps to skip, comma delimited]") return False targets = self.getTarget(pair[0]) if targets == None: logger.writeError("Target not found in -s command-line option: " + pair[0]) return False targets.skipSteps = pair[1].split(",") return True
def callPythonCommand(namespace, function, target, options, project): filename = namespace + ".py" projectPath = os.path.abspath(".") if not projectPath in sys.path: sys.path.insert(0, projectPath) mixDownPath = os.path.dirname(sys.argv[0]) mdPath = os.path.join(mixDownPath, "md") if os.path.exists(mdPath) and not mdPath in sys.path: sys.path.insert(0, mdPath) stepsPath = os.path.dirname(os.path.abspath(steps.__file__)) if mdPath != stepsPath and not stepsPath in sys.path: sys.path.insert(0, stepsPath) namespaceFound = False for currPath in sys.path: currNamespacePath = os.path.join(currPath, filename) if os.path.exists( currNamespacePath) and not os.path.isdir(currNamespacePath): namespaceFound = True break if not namespaceFound: logger.writeError("Could not find python file in path.", filePath=filename) return False importedNamespace = __import__(namespace) try: target.pythonCallInfo.success = False target.pythonCallInfo.target = target target.pythonCallInfo.options = options target.pythonCallInfo.project = project pythonCallInfo = getattr(importedNamespace, function)(target.pythonCallInfo) except AttributeError as e: logger.writeError(str(e)) traceback.print_exc(file=logger.getErrorFd()) return False if not pythonCallInfo.success: return False target = pythonCallInfo.target options = pythonCallInfo.options project = pythonCallInfo.project return True
def callPythonCommand(namespace, function, target, options, project): filename = namespace + ".py" projectPath = os.path.abspath(".") if not projectPath in sys.path: sys.path.insert(0, projectPath) mixDownPath = os.path.dirname(sys.argv[0]) mdPath = os.path.join(mixDownPath, "md") if os.path.exists(mdPath) and not mdPath in sys.path: sys.path.insert(0, mdPath) stepsPath = os.path.dirname(os.path.abspath(steps.__file__)) if mdPath != stepsPath and not stepsPath in sys.path: sys.path.insert(0, stepsPath) namespaceFound = False for currPath in sys.path: currNamespacePath = os.path.join(currPath, filename) if os.path.exists(currNamespacePath) and not os.path.isdir(currNamespacePath): namespaceFound = True break if not namespaceFound: logger.writeError("Could not find python file in path.", filePath=filename) return False importedNamespace = __import__(namespace) try: target.pythonCallInfo.success = False target.pythonCallInfo.target = target target.pythonCallInfo.options = options target.pythonCallInfo.project = project pythonCallInfo = getattr(importedNamespace, function)(target.pythonCallInfo) except AttributeError as e: logger.writeError(str(e)) traceback.print_exc(file=logger.getErrorFd()) return False if not pythonCallInfo.success: return False target = pythonCallInfo.target options = pythonCallInfo.options project = pythonCallInfo.project return True
def __assignDepthToTargetList(self): q = Queue.Queue() for targets in self.targets: q.put(targets.name) while not q.empty(): currName = q.get() currTarget = self.getTarget(currName) for currChildName in currTarget.dependsOn: currChildTarget = self.getTarget(currChildName) if not currChildTarget: logger.writeError("Target '" + currTarget.name + "' has non-existant dependency '" + currChildName + "'", "", self.path, exitProgram=True) if currChildTarget.dependencyDepth < ( currTarget.dependencyDepth + 1): currChildTarget.dependencyDepth = currTarget.dependencyDepth + 1 q.put(currChildName)
def __validateDependsOnLists(self): if len(self.targets) == 0: return True for currTarget in self.targets: normalizedName = target.normalizeName(currTarget.name) checkedDependencies = [] for dependency in currTarget.dependsOn: normalizedDepedancy = target.normalizeName(dependency) if normalizedDepedancy == normalizedName: logger.writeError("Target cannot depend on itself", currTarget.name, "", self.path) return False if normalizedDepedancy in checkedDependencies: logger.writeError( "Target has duplicate dependency '" + dependency + "'", currTarget.name, "", self.path) return False if self.getTarget(dependency) is None: logger.writeError( "Target has non-existant dependency '" + dependency + "'", currTarget.name, "", self.path) return False checkedDependencies.append(normalizedDepedancy) path = [target.normalizeName(self.targets[0].name)] return self.__searchPathsForCycles(path)
def addSkipStepFromOptions(self, options): if options.skipSteps == '' or options.skipSteps == None: return True skipSteps = options.skipSteps for item in skipSteps.split(","): pair = item.split(":") if len(pair) != 2 or pair[0] == "" or pair[1] == "": logger.writeError("Invalid command-line -s pair found: " + item) logger.writeMessage( "Proper use: -s[Semi-colon delimited list of Skip Step pairs]" ) logger.writeMessage( "Skip Step Pair: [targetName]:[Steps to skip, comma delimited]" ) return False targets = self.getTarget(pair[0]) if targets == None: logger.writeError( "Target not found in -s command-line option: " + pair[0]) return False targets.skipSteps = pair[1].split(",") return True
def __processProfileCommandline(self, commandline): if len(commandline) < 2: self.printUsage() return False self.verbose = True for currArg in commandline[1:]: if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"): self.printUsage() return False elif currArg == "--profile": continue currFlag = currArg[:2].lower() currValue = currArg[2:] if currFlag in ("-p", "-l", "-n", "-j", "-b", "-w", "-k", "-o", "-g", "-i", "-v", "-f"): logger.writeError( "Command-line option is not allowed in profile mode: " + currArg) return False elif currFlag == "-d": if not validateOptionPair(currFlag, currValue): return False paths = currValue.split(",") for path in paths: self.overrideSearchPath.append((path, False)) elif currFlag == "-r": if not validateOptionPair(currFlag, currValue): return False paths = currValue.split(",") for path in paths: self.overrideSearchPath.append((path, True)) elif os.path.splitext(currArg)[1] == ".mdo": if os.path.isfile(currArg): logger.writeError( "Given override file name already exists: " + currArg) return False self.overrideFile = currArg else: logger.writeError( "File not found or command-line option not understood: " + currArg) return False if len(self.overrideSearchPath) == 0: self.overrideSearchPath = [("/usr/local/bin", False), ("/usr/bin", False)] if self.overrideFile == "": self.overrideFile = os.path.join(os.getcwd(), socket.gethostname() + ".mdo") return True
def __processImportCommandline(self, commandline): if len(commandline) < 3: self.printUsage() return False for currArg in commandline[1:]: if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"): self.printUsage() return False elif currArg == "--import": continue elif utilityFunctions.isURL(currArg) or os.path.isfile( currArg) or os.path.isdir(currArg): name = target.targetPathToName(currArg) if name == "": return False currTarget = target.Target(name, currArg) self.targetsToImport.append(currTarget) continue currFlag = currArg[:2].lower() currValue = currArg[2:] if currFlag in ("-p", "-l", "-n", "-j", "-b", "-w", "-k", "-o", "-g", "-f"): logger.writeError( "Command-line option is not allowed in import mode: " + currArg) return False if currFlag == "-i": if not validateOption(currFlag, currValue): return False self.interactive = True elif currFlag == "-v": if not validateOption(currFlag, currValue): return False self.verbose = True else: logger.writeError( "File not found or command-line option not understood: " + currArg) return False if len(self.targetsToImport) == 0: self.printUsage() logger.writeError( "No packages given when MixDown is in import mode") return False return True
def validate(self): if not self.__validateOptionsDir(self.buildDir) or not\ self.__validateOptionsDir(self.downloadDir) or not\ self.__validateOptionsDir(self.logDir): return False if self.importMode and self.cleanMode: logger.writeError("MixDown cannot be in both import mode and clean mode") return False if self.overrideFile == "" and len(self.overrideGroupNames) == 0: logger.writeError("Override group name(s) given but no override file specified. Use command-line option '-o<filename>'.") return False if self.overrideFile != "" and len(self.overrideGroupNames) == 0: logger.writeError("Override file given but no override group(s) specified. Use command-line option '-g<Name>[,<Name>...]'.") return False return True
def __processProfileCommandline(self, commandline): if len(commandline) < 2: self.printUsage() return False self.verbose = True for currArg in commandline[1:]: if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"): self.printUsage() return False elif currArg == "--profile": continue currFlag = currArg[:2].lower() currValue = currArg[2:] if currFlag in ("-p", "-l", "-n", "-j", "-b", "-w", "-k", "-o", "-g", "-i", "-v", "-f"): logger.writeError("Command-line option is not allowed in profile mode: " + currArg) return False elif currFlag == "-d": if not validateOptionPair(currFlag, currValue): return False paths = currValue.split(",") for path in paths: self.overrideSearchPath.append((path, False)) elif currFlag == "-r": if not validateOptionPair(currFlag, currValue): return False paths = currValue.split(",") for path in paths: self.overrideSearchPath.append((path, True)) elif os.path.splitext(currArg)[1] == ".mdo": if os.path.isfile(currArg): logger.writeError("Given override file name already exists: " + currArg) return False self.overrideFile = currArg else: logger.writeError("File not found or command-line option not understood: " + currArg) return False if len(self.overrideSearchPath) == 0: self.overrideSearchPath = [("/usr/local/bin", False), ("/usr/bin", False)] if self.overrideFile == "": self.overrideFile = os.path.join(os.getcwd(), socket.gethostname() + ".mdo") return True
def __processImportCommandline(self, commandline): if len(commandline) < 3: self.printUsage() return False for currArg in commandline[1:]: if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"): self.printUsage() return False elif currArg == "--import": continue elif utilityFunctions.isURL(currArg) or os.path.isfile(currArg) or os.path.isdir(currArg): name = target.targetPathToName(currArg) if name == "": return False currTarget = target.Target(name, currArg) self.targetsToImport.append(currTarget) continue currFlag = currArg[:2].lower() currValue = currArg[2:] if currFlag in ("-p", "-l", "-n", "-j", "-b", "-w", "-k", "-o", "-g", "-f"): logger.writeError("Command-line option is not allowed in import mode: " + currArg) return False if currFlag == "-i": if not validateOption(currFlag, currValue): return False self.interactive = True elif currFlag == "-v": if not validateOption(currFlag, currValue): return False self.verbose = True else: logger.writeError("File not found or command-line option not understood: " + currArg) return False if len(self.targetsToImport) == 0: self.printUsage() logger.writeError("No packages given when MixDown is in import mode") return False return True
def validate(self): if not self.__validateOptionsDir(self.buildDir) or not\ self.__validateOptionsDir(self.downloadDir) or not\ self.__validateOptionsDir(self.logDir): return False if self.importMode and self.cleanMode: logger.writeError( "MixDown cannot be in both import mode and clean mode") return False if self.overrideFile == "" and len(self.overrideGroupNames) == 0: logger.writeError( "Override group name(s) given but no override file specified. Use command-line option '-o<filename>'." ) return False if self.overrideFile != "" and len(self.overrideGroupNames) == 0: logger.writeError( "Override file given but no override group(s) specified. Use command-line option '-g<Name>[,<Name>...]'." ) return False return True
def __validateDependsOnLists(self): if len(self.targets) == 0: return True for currTarget in self.targets: normalizedName = target.normalizeName(currTarget.name) checkedDependencies = [] for dependency in currTarget.dependsOn: normalizedDepedancy = target.normalizeName(dependency) if normalizedDepedancy == normalizedName: logger.writeError("Target cannot depend on itself", currTarget.name, "", self.path) return False if normalizedDepedancy in checkedDependencies: logger.writeError("Target has duplicate dependency '" + dependency + "'", currTarget.name, "", self.path) return False if self.getTarget(dependency) is None: logger.writeError("Target has non-existant dependency '" + dependency + "'", currTarget.name, "", self.path) return False checkedDependencies.append(normalizedDepedancy) path = [target.normalizeName(self.targets[0].name)] return self.__searchPathsForCycles(path)
def read(self): if self.path == "": logger.writeError("No project file was specified") return False f = open(self.path, "r") try: currTarget = None lineCount = 0 for currLine in f: lineCount += 1 lastPackageLineNumber = 0 currLine = str.strip(currLine) if (currLine == "") or currLine.startswith('#') or currLine.startswith('//'): pass else: colonIndex = currLine.find(":") equalsIndex = currLine.find("=") if (colonIndex == -1) and (equalsIndex == -1): logger.writeError("Line could not be understood by project parser. Lines can be comments, defines, or target information.", "", "", self.path, lineCount) return False if ((colonIndex == -1) and (equalsIndex != -1)) or\ ((colonIndex != -1) and (equalsIndex != -1) and (equalsIndex < colonIndex)): currPair = currLine.split("=", 1) currPair = currPair[0].strip(), currPair[1].strip() currName = str.lower(currPair[0]) self.defines[currPair[0]] = currPair[1] else: currPair = currLine.split(":", 1) currPair = currPair[0].strip(), currPair[1].strip() currName = str.lower(currPair[0]) if (currName != "name") and (currTarget is None): logger.writeError("'" + currPair[0] + "' declared before 'name' in Project file", "", "", self.path, lineCount) return False if currName == "name": lastPackageLineNumber = lineCount if currTarget != None: if not self.__addTarget(currTarget, lastPackageLineNumber): return False currTarget = target.Target(currPair[1]) elif currName == "path": if currTarget.path != "": logger.writeError("Project targets can only have one 'Path' defined", "", "", self.path, lineCount) return False fullPath = os.path.abspath(currPair[1]) currTarget.origPath = fullPath currTarget.path = fullPath elif currName == "output": if currTarget.outputPathSpecified: logger.writeError("Project targets can only have one 'Output' defined", "", "", self.path, lineCount) return False currTarget.outputPath = currPair[1] currTarget.outputPathSpecified = True elif currName == "dependson": if currTarget.dependsOn != []: logger.writeError("Project targets can only have one 'DependsOn' defined (use a comma delimited list for multiple dependencies)", "", "", self.path, lineCount) return False if currPair[1] != "": dependsOnList = utilityFunctions.stripItemsInList(currPair[1].split(",")) normalizedName = target.normalizeName(currTarget.name) for dependency in dependsOnList: if not re.match(r'[\w\.+_\-]+$', dependency): logger.writeError("Dependancy name, '" + dependency + "', found with invalid character. Only alphanumeric is allowed.", currTarget.name, "", self.path, lineCount) return False if target.normalizeName(dependency) == normalizedName: logger.writeError("Project targets cannot depend on themselves", currTarget.name, "", self.path, lineCount) return False currTarget.dependsOn = dependsOnList elif currName == "aliases": if currTarget.aliases != []: logger.writeError("Project targets can only have one 'Aliases' defined (use a comma delimited list for multiple aliases)", "", "", self.path, lineCount) return False if currPair[1] != "": aliases = utilityFunctions.stripItemsInList(currPair[1].split(",")) noralizedName = target.normalizeName(currTarget.name) for alias in aliases: if target.normalizeName(alias) == normalizedName: logger.writeError("Project target alias cannot be same as its name", currTarget.name, "", self.path, lineCount) return False currTarget.aliases = aliases elif currName == "skipsteps" or currName == "skipstep": if currTarget.skipSteps != []: logger.writeError("Project targets can only have one 'SkipSteps' defined (use a comma delimited list for multiple steps)", "", "", self.path, lineCount) return False currTarget.skipSteps = utilityFunctions.stripItemsInList(str.lower(currPair[1]).split(",")) elif currName == "prefix": if currTarget.prefix != "": logger.writeError("Project targets can only have one 'Prefix' defined", "", "", self.path, lineCount) return False currTarget.prefix = currPair[1] elif currName == "version": if currTarget.version != "": logger.writeError("Project targets can only have one 'Version' defined", "", "", self.path, lineCount) return False currTarget.version = currPair[1] elif currName in commands.buildSteps: if currTarget.findBuildStep(currName) != None: logger.writeError("Project targets can only have one '" + currName + "' defined", "", "", self.path, lineCount) return False currTarget.buildSteps.append(commands.BuildStep(currName, currPair[1])) else: logger.writeError("Cannot understand given line: '" + currLine + "'", "", "", self.path, lineCount) return False if not self.__addTarget(currTarget, lastPackageLineNumber): return False finally: f.close() return True
def processCommandline(self, commandline=[]): if len(commandline) < 2: self.printUsage() return False #Find mode for arg in commandline[1:]: #skip script name loweredArg = arg.lower() if loweredArg == "--import": self.importMode = True elif loweredArg == "--clean": self.cleanMode = True self.cleanMixDown = False elif loweredArg == "--profile": self.profileMode = True if (self.cleanMode and (self.profileMode or self.importMode)) or\ (self.profileMode and (self.importMode or self.cleanMode)): logger.writeError( "MixDown cannot be in both two command-line modes at the same time. Run 'MixDown --help' for instructions." ) return False if self.profileMode: return self.__processProfileCommandline(commandline) if self.importMode: return self.__processImportCommandline(commandline) for currArg in commandline[1:]: #skip script name #Handle all options that don't follow -<letter><option> if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"): self.printUsage() return False elif os.path.splitext(currArg)[1] == ".md": if not os.path.isfile(currArg): logger.writeError("Project file " + currArg + " does not exist") return False else: self.projectFile = currArg continue elif currArg.lower() == "--clean": continue elif currArg.lower().startswith("--pythonpath="): sys.path.append(os.path.abspath(currArg[13:])) continue #Handle all options that follow -<letter><option> currFlag = currArg[:2].lower() currValue = currArg[2:] if currFlag in ("-i"): logger.writeError( "Command-line option is not allowed in build or clean mode: " + currArg) return False elif currFlag == "-p": if not validateOptionPair(currFlag, currValue): return False defines.setPrefixDefines(self.defines, os.path.abspath(currValue)) self.prefixDefined = True elif currFlag == "-t": if not validateOptionPair(currFlag, currValue): return False try: count = int(currValue) except ValueError: count = 0 if count < 1: logger.writeError("Positive numeric value needed " + definePair) return False self.threadCount = count elif currFlag == "-j": if not validateOptionPair(currFlag, currValue): return False try: count = int(currValue) except ValueError: count = 0 if count < 1: logger.writeError("Positive numeric value needed " + definePair) return False #Add "-j<jobSlots>" only if user defines -j on command-line defines.setJobSlotsDefines(self.defines, currValue) elif currFlag == "-l": if not validateOptionPair(currFlag, currValue): return False self.logger = str.lower(currValue) elif currFlag == "-r": if not validateOption(currFlag, currValue): return False self.restart = True elif currFlag == "-c": if not validateOption(currFlag, currValue): return False self.continueBuilding = True elif currFlag == "-b": if not validateOptionPair(currFlag, currValue): return False self.buildDir = currValue elif currFlag == "-w": if not validateOptionPair(currFlag, currValue): return False self.downloadDir = currValue elif currFlag == "-f": if not validateOptionPair(currFlag, currValue): return False self.logDir = currValue elif currFlag == "-k": if not validateOption(currFlag, currValue): return False if self.cleanMode: logger.writeError( "Command-line arguments '--clean' and '-k' cannot both be used at the same time" ) return False self.cleanMixDown = False elif currFlag == "-v": if not validateOption(currFlag, currValue): return False self.verbose = True elif currFlag == "-s": if not validateOptionPair(currFlag, currValue): return False self.skipSteps = currValue elif currFlag == "-o": if not validateOptionPair(currFlag, currValue): return False self.overrideFile = currValue elif currFlag == "-d": if not validateOptionPair(currFlag, currValue): return False for definePair in currValue.split(","): splitPair = definePair.split("=") if len(splitPair) != 2: logger.writeError("Invalid define pair given, " + definePair) return False if splitPair[0].lower() in self.commandLineDefines: logger.writeError("Define pair already given, " + definePair) return False self.commandLineDefines[splitPair[0]] = splitPair[1] elif currFlag == "-n": if not validateOptionPair(currFlag, currValue): return False names = currValue.split(",") if len(names) == 0: logger.writeError( "Build target names command-line option used but no names specified." ) return False for name in names: currName = name.lower().strip() if currName == "": logger.writeError( "Empty build target name given on command-line (option '-n')" ) return False self.targetsToBuild.append(currName) elif currFlag == "-g": if not validateOptionPair(currFlag, currValue): return False names = currValue.split(",") if len(names) == 0: logger.writeError( "Override group command-line option used but no names specified." ) return False for name in names: currName = name.lower().strip() if currName == "": logger.writeError( "Empty override group name given on command-line (option '-g')" ) return False self.overrideGroupNames.append(currName) else: logger.writeError("Command-line argument '" + currArg + "' not understood") return False return True
def validateOptionPair(flag, value): if value == "": logger.writeError(flag + " option requires a following value") return False return True
def validateOption(flag, value): if value != "": logger.writeError(flag + " option does not require a following value") return False return True
def readStatusLog(self, options): lineCount = 0 if not os.path.exists(options.statusLogPath) or not os.path.isfile(options.statusLogPath): return True, False fd = open(options.statusLogPath, 'r') try: #path:<self.path> l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount) if EOF: return False, True if len(l) != 2: logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount) return False, True if not self.__assureEqualsInStatusLog(l, "path", os.path.abspath(self.path)): return False, True #buildDir:<options.buildDir> l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount) if EOF: return False, True if len(l) != 2: logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount) return False, True if not self.__assureEqualsInStatusLog(l, "buildDir", os.path.abspath(options.buildDir)): return False, True #prefix:<prefix>:<defined> l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount) if EOF: return False, True if len(l) != 3: logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount) return False, True if not self.__assureEqualsInStatusLog(l, "prefix", os.path.abspath(options.defines[defines.mdPrefix[0]])): return False, True if l[2] != str(options.prefixDefined): return False, True #Target groups: one target line followed by multiple step lines #<target.name>:<target.origPath>:<success> #<step.name>:<step.command>:<step.restartPath>:<step.success> while True: l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount) if EOF: break if len(l) == 3: currTarget = self.getTarget(l[0]) if currTarget == None: logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount) continue currTarget.origPath = l[1] currTarget.success = utilityFunctions.boolToStr(l[2]) elif len(l) == 4: currTarget = self.getTarget(l[0]) if currTarget == None: continue currStep = currTarget.findBuildStep(l[0]) if currStep == None: continue currStep.restartPath = l[2] if currStep.restartPath == "": currStep.restartPath = currTarget.origPath currStep.success = utilityFunctions.boolToStr(l[3]) if currStep.command != l[1]: currStep.success = False else: logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount) return True, True finally: fd.close()
def readStatusLog(self, options): lineCount = 0 if not os.path.exists(options.statusLogPath) or not os.path.isfile( options.statusLogPath): return True, False fd = open(options.statusLogPath, 'r') try: #path:<self.path> l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount) if EOF: return False, True if len(l) != 2: logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount) return False, True if not self.__assureEqualsInStatusLog(l, "path", os.path.abspath(self.path)): return False, True #buildDir:<options.buildDir> l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount) if EOF: return False, True if len(l) != 2: logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount) return False, True if not self.__assureEqualsInStatusLog( l, "buildDir", os.path.abspath(options.buildDir)): return False, True #prefix:<prefix>:<defined> l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount) if EOF: return False, True if len(l) != 3: logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount) return False, True if not self.__assureEqualsInStatusLog( l, "prefix", os.path.abspath(options.defines[defines.mdPrefix[0]])): return False, True if l[2] != str(options.prefixDefined): return False, True #Target groups: one target line followed by multiple step lines #<target.name>:<target.origPath>:<success> #<step.name>:<step.command>:<step.restartPath>:<step.success> while True: l, lineCount, EOF = self.__getStatusLogLine(fd, lineCount) if EOF: break if len(l) == 3: currTarget = self.getTarget(l[0]) if currTarget == None: logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount) continue currTarget.origPath = l[1] currTarget.success = utilityFunctions.boolToStr(l[2]) elif len(l) == 4: currTarget = self.getTarget(l[0]) if currTarget == None: continue currStep = currTarget.findBuildStep(l[0]) if currStep == None: continue currStep.restartPath = l[2] if currStep.restartPath == "": currStep.restartPath = currTarget.origPath currStep.success = utilityFunctions.boolToStr(l[3]) if currStep.command != l[1]: currStep.success = False else: logger.writeError("Unknown line in status log:\n" + ':'.join(l), filePath="options.statusLogPath", lineNumber=lineCount) return True, True finally: fd.close()
def read(self): if self.path == "": logger.writeError("No project file was specified") return False f = open(self.path, "r") try: currTarget = None lineCount = 0 for currLine in f: lineCount += 1 lastPackageLineNumber = 0 currLine = str.strip(currLine) if (currLine == "" ) or currLine.startswith('#') or currLine.startswith('//'): pass else: colonIndex = currLine.find(":") equalsIndex = currLine.find("=") if (colonIndex == -1) and (equalsIndex == -1): logger.writeError( "Line could not be understood by project parser. Lines can be comments, defines, or target information.", "", "", self.path, lineCount) return False if ((colonIndex == -1) and (equalsIndex != -1)) or\ ((colonIndex != -1) and (equalsIndex != -1) and (equalsIndex < colonIndex)): currPair = currLine.split("=", 1) currPair = currPair[0].strip(), currPair[1].strip() currName = str.lower(currPair[0]) self.defines[currPair[0]] = currPair[1] else: currPair = currLine.split(":", 1) currPair = currPair[0].strip(), currPair[1].strip() currName = str.lower(currPair[0]) if (currName != "name") and (currTarget is None): logger.writeError( "'" + currPair[0] + "' declared before 'name' in Project file", "", "", self.path, lineCount) return False if currName == "name": lastPackageLineNumber = lineCount if currTarget != None: if not self.__addTarget( currTarget, lastPackageLineNumber): return False currTarget = target.Target(currPair[1]) elif currName == "path": if currTarget.path != "": logger.writeError( "Project targets can only have one 'Path' defined", "", "", self.path, lineCount) return False fullPath = os.path.abspath(currPair[1]) currTarget.origPath = fullPath currTarget.path = fullPath elif currName == "output": if currTarget.outputPathSpecified: logger.writeError( "Project targets can only have one 'Output' defined", "", "", self.path, lineCount) return False currTarget.outputPath = currPair[1] currTarget.outputPathSpecified = True elif currName == "dependson": if currTarget.dependsOn != []: logger.writeError( "Project targets can only have one 'DependsOn' defined (use a comma delimited list for multiple dependencies)", "", "", self.path, lineCount) return False if currPair[1] != "": dependsOnList = utilityFunctions.stripItemsInList( currPair[1].split(",")) normalizedName = target.normalizeName( currTarget.name) for dependency in dependsOnList: if not re.match(r'[\w\.+_\-]+$', dependency): logger.writeError( "Dependancy name, '" + dependency + "', found with invalid character. Only alphanumeric is allowed.", currTarget.name, "", self.path, lineCount) return False if target.normalizeName( dependency) == normalizedName: logger.writeError( "Project targets cannot depend on themselves", currTarget.name, "", self.path, lineCount) return False currTarget.dependsOn = dependsOnList elif currName == "aliases": if currTarget.aliases != []: logger.writeError( "Project targets can only have one 'Aliases' defined (use a comma delimited list for multiple aliases)", "", "", self.path, lineCount) return False if currPair[1] != "": aliases = utilityFunctions.stripItemsInList( currPair[1].split(",")) noralizedName = target.normalizeName( currTarget.name) for alias in aliases: if target.normalizeName( alias) == normalizedName: logger.writeError( "Project target alias cannot be same as its name", currTarget.name, "", self.path, lineCount) return False currTarget.aliases = aliases elif currName == "skipsteps" or currName == "skipstep": if currTarget.skipSteps != []: logger.writeError( "Project targets can only have one 'SkipSteps' defined (use a comma delimited list for multiple steps)", "", "", self.path, lineCount) return False currTarget.skipSteps = utilityFunctions.stripItemsInList( str.lower(currPair[1]).split(",")) elif currName == "prefix": if currTarget.prefix != "": logger.writeError( "Project targets can only have one 'Prefix' defined", "", "", self.path, lineCount) return False currTarget.prefix = currPair[1] elif currName == "version": if currTarget.version != "": logger.writeError( "Project targets can only have one 'Version' defined", "", "", self.path, lineCount) return False currTarget.version = currPair[1] elif currName in commands.buildSteps: if currTarget.findBuildStep(currName) != None: logger.writeError( "Project targets can only have one '" + currName + "' defined", "", "", self.path, lineCount) return False currTarget.buildSteps.append( commands.BuildStep(currName, currPair[1])) else: logger.writeError( "Cannot understand given line: '" + currLine + "'", "", "", self.path, lineCount) return False if not self.__addTarget(currTarget, lastPackageLineNumber): return False finally: f.close() return True
def importTargets(options): logger.setLogger("console") finalTargets = [] ignoredTargets = [] partialImport = False fetchStep = commands.BuildStep("fetch", commands.__getFetchCommand(None)) unpackStep = commands.BuildStep("unpack", commands.__getUnpackCommand(None)) tempDir = tempfile.mkdtemp(prefix="mixdown-") options.downloadDir = os.path.join(tempDir, "mdDownloads") while len(options.targetsToImport) != 0: target = options.targetsToImport.pop(0) logger.writeMessage("Analyzing target...", target.name) logger.writeMessage("Extracting target...", target.name) target.outputPath = os.path.join(tempDir, target.name) target.currBuildStep = fetchStep if not commands.buildStepActor(target, options, None): utilityFunctions.removeDir(tempDir) return None, False target.currBuildStep = unpackStep if not commands.buildStepActor(target, options, None): utilityFunctions.removeDir(tempDir) return None, False #Generate build files and find possible dependencies possibleDeps = [] if cmake.isCMakeProject(target.path): logger.writeMessage("CMake project found...", target.name) logger.writeMessage("Analyzing for dependencies...", target.name) possibleDeps = cmake.getDependencies(target.path, target.name) elif autoTools.isAutoToolsProject(target.path): logger.writeMessage("Auto Tools project found...", target.name) if not os.path.exists(os.path.join(target.path, "configure")): if not autoTools.generateConfigureFiles(target.path, target.name): utilityFunctions.removeDir(tempDir) return None, False logger.writeMessage("Analyzing for dependencies...", target.name) possibleDeps = autoTools.getDependencies(target.path, target.name) if possibleDeps == None: target.comment = "Unable to parse 'configure --help' output. MixDown cannot determine dependencies for this target." logger.writeError(target.comment, target.name) partialImport = True possibleDeps = [] elif make.isMakeProject(target.path): target.comment = "Make project found. MixDown cannot determine dependencies from Make projects." logger.writeError(target.comment, target.name) partialImport = True else: target.comment = "Unknown build system found. MixDown cannot determine dependencies or build commands." logger.writeError(target.comment, target.name) partialImport = True #Find actual dependencies for possibleDependency in possibleDeps: if getTarget(possibleDependency, finalTargets + options.targetsToImport): logger.writeMessage("Known dependency found (" + possibleDependency + ")", target.name) target.dependsOn.append(possibleDependency) continue elif options.interactive and possibleDependency in ignoredTargets: logger.writeMessage("Previously ignored dependency found (" + possibleDependency + ")", target.name) continue if searchForPossibleAliasInList(possibleDependency, finalTargets + options.targetsToImport, options.interactive): target.dependsOn.append(possibleDependency) elif not options.interactive: logger.writeMessage("Ignoring unknown dependency (" + possibleDependency + ")", target.name) else: logger.writeMessage("Unknown dependency found (" + possibleDependency + ")", target.name) userInput = raw_input(possibleDependency + ": Input location, target name, or blank to ignore:").strip() if userInput == "": ignoredTargets.append(possibleDependency) elif os.path.isfile(userInput) or os.path.isdir(userInput) or utilityFunctions.isURL(userInput): name = target.targetPathToName(userInput) if name == "": return None, False newTarget = target.Target(name, userInput) options.targetsToImport.append(newTarget) if target.normalizeName(possibleDependency) != target.normalizeName(userInput): newTarget.aliases.append(possibleDependency) target.dependsOn.append(possibleDependency) else: aliasTarget = getTarget(userInput, finalTargets + options.targetsToImport, possibleDependency) if aliasTarget != None: logger.writeMessage("Alias added (" + userInput + ")", aliasTarget.name) target.dependsOn.append(possibleDependency) else: aliasLocation = raw_input(userInput + ": Target name not found in any known targets. Location of new target:").strip() if os.path.isfile(aliasLocation) or os.path.isdir(aliasLocation) or utilityFunctions.isURL(aliasLocation): name = target.targetPathToName(aliasLocation) if name == "": return None, False newTarget = target.Target(name, aliasLocation) notReviewedTargets.append(newTarget) if target.normalizeName(possibleDependency) != target.normalizeName(aliasLocation): newTarget.aliases.append(possibleDependency) target.dependsOn.append(possibleDependency) else: logger.writeError(userInput + ": Alias location not understood.", exitProgram=True) finalTargets.append(target) #Create project for targets projects = project.Project("ProjectNameNotDetermined", finalTargets) if not projects.examine(options): logger.writeError("Project failed examination", exitProgram=True) if not projects.validate(options): logger.writeError("Project failed validation", exitProgram=True) mainTargetPath = projects.targets[0].origPath if utilityFunctions.isURL(mainTargetPath): mainTargetPath = utilityFunctions.URLToFilename(mainTargetPath) mainTargetName, mainTargetVersion = utilityFunctions.splitFileName(mainTargetPath) if mainTargetVersion != "": projects.name = mainTargetName + "-" + mainTargetVersion else: projects.name = mainTargetName projects.path = projects.name + ".md" for target in projects.targets: target.outputPath = "" if projects.examine(options): logger.writeMessage("\nFinal targets...\n\n" + str(projects)) projects.write() utilityFunctions.removeDir(tempDir) return projects, partialImport
def __validateOptionsDir(self, path): if os.path.isfile(self.buildDir): logger.writeError("Cannot create directory by MixDown, a file by the same name already exists: " + path) return False return True
def readGroups(filename): if not os.path.exists(filename): logger.writeError( "Given Override file path did not exist. Check your -o command-line option.", filePath=filename) return None groups = list() overrideGroup = None tokenizer = Tokenizer(filename) if not tokenizer.tokenize(): return None tokens = tokenizer.tokens i = 0 lengthOfTokens = len(tokens) while i < lengthOfTokens: #Syntax start: <name>[, <name>]* overrideGroup = OverrideGroup() while not (tokens[i].type == TokenType.Symbol and tokens[i].value == '{'): if len(overrideGroup.name) > 50: logger.writeError("Override group name exceeded limit (50).", filePath=filename) return None if tokens[i].type == TokenType.Identifier: overrideGroup.name.append(tokens[i].value.lower()) i += 1 if i >= lengthOfTokens: logger.writeError( "Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None if tokens[i].type == TokenType.Symbol and tokens[ i].value == '{': #End of override group name syntax hit break elif tokens[i].type == TokenType.Symbol and tokens[ i].value == ',': i += 1 if i >= lengthOfTokens: logger.writeError( "Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None else: logger.writeError("Expected ',' got '" + tokens[i].value + "'", filePath=filename) return None else: logger.writeError("Expected Override group name, got '" + tokens[i].value + "'", filePath=filename) return None #Syntax end #Syntax start: #{ # <Identifier> = <string> # ... # <Identifier> = <string> #} if not (tokens[i].type == TokenType.Symbol and tokens[i].value == '{'): logger.writeError("Expected '{' got '" + tokens[i].value + "'", filePath=filename) return None i += 1 if i >= lengthOfTokens: logger.writeError( "Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None while True: if tokens[i].type == TokenType.Symbol and tokens[i].value == '}': i += 1 break overrideName = "" overrideValue = "" #Syntax start: <Identifier> = <string> if not tokens[i].type == TokenType.Identifier: logger.writeError("Expected Override identifier got '" + tokens[i].value + "'", filePath=filename) return None overrideNameOriginal = tokens[i].value overrideName = tokens[i].value.lower() i += 1 if i >= lengthOfTokens: logger.writeError( "Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None if not (tokens[i].type == TokenType.Symbol and tokens[i].value == '='): logger.writeError("Expected '=' got '" + tokens[i].value + "'", filePath=filename) return None i += 1 if i >= lengthOfTokens: logger.writeError( "Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None if not tokens[i].type == TokenType.String: logger.writeError("Expected Override value string got '" + tokens[i].value + "'", filePath=filename) return None overrideString = tokens[i].value i += 1 if i >= lengthOfTokens: logger.writeError( "Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None overrideGroup[overrideName] = overrideString #Syntax end #Syntax end overrideGroupNames = ", ".join(overrideGroup) for group in groups: if ", ".join(group.name) == overrideGroupNames: logger.writeError("Duplicate override group name found: " + overrideGroupNames, filePath=options.overrideFile) return None groups.append(overrideGroup) return groups
def processCommandline(self, commandline=[]): if len(commandline) < 2: self.printUsage() return False #Find mode for arg in commandline[1:]: #skip script name loweredArg = arg.lower() if loweredArg == "--import": self.importMode = True elif loweredArg == "--clean": self.cleanMode = True self.cleanMixDown = False elif loweredArg == "--profile": self.profileMode = True if (self.cleanMode and (self.profileMode or self.importMode)) or\ (self.profileMode and (self.importMode or self.cleanMode)): logger.writeError("MixDown cannot be in both two command-line modes at the same time. Run 'MixDown --help' for instructions.") return False if self.profileMode: return self.__processProfileCommandline(commandline) if self.importMode: return self.__processImportCommandline(commandline) for currArg in commandline[1:]: #skip script name #Handle all options that don't follow -<letter><option> if currArg.lower() in ("/help", "/h", "-help", "--help", "-h"): self.printUsage() return False elif os.path.splitext(currArg)[1] == ".md": if not os.path.isfile(currArg): logger.writeError("Project file " + currArg + " does not exist") return False else: self.projectFile = currArg continue elif currArg.lower() == "--clean": continue elif currArg.lower().startswith("--pythonpath="): sys.path.append(os.path.abspath(currArg[13:])) continue #Handle all options that follow -<letter><option> currFlag = currArg[:2].lower() currValue = currArg[2:] if currFlag in ("-i"): logger.writeError("Command-line option is not allowed in build or clean mode: " + currArg) return False elif currFlag == "-p": if not validateOptionPair(currFlag, currValue): return False defines.setPrefixDefines(self.defines, os.path.abspath(currValue)) self.prefixDefined = True elif currFlag == "-t": if not validateOptionPair(currFlag, currValue): return False try: count = int(currValue) except ValueError: count = 0 if count < 1: logger.writeError("Positive numeric value needed " + definePair) return False self.threadCount = count elif currFlag == "-j": if not validateOptionPair(currFlag, currValue): return False try: count = int(currValue) except ValueError: count = 0 if count < 1: logger.writeError("Positive numeric value needed " + definePair) return False #Add "-j<jobSlots>" only if user defines -j on command-line defines.setJobSlotsDefines(self.defines, currValue) elif currFlag == "-l": if not validateOptionPair(currFlag, currValue): return False self.logger = str.lower(currValue) elif currFlag == "-r": if not validateOption(currFlag, currValue): return False self.restart = True elif currFlag == "-c": if not validateOption(currFlag, currValue): return False self.continueBuilding = True elif currFlag == "-b": if not validateOptionPair(currFlag, currValue): return False self.buildDir = currValue elif currFlag == "-w": if not validateOptionPair(currFlag, currValue): return False self.downloadDir = currValue elif currFlag == "-f": if not validateOptionPair(currFlag, currValue): return False self.logDir = currValue elif currFlag == "-k": if not validateOption(currFlag, currValue): return False if self.cleanMode: logger.writeError("Command-line arguments '--clean' and '-k' cannot both be used at the same time") return False self.cleanMixDown = False elif currFlag == "-v": if not validateOption(currFlag, currValue): return False self.verbose = True elif currFlag == "-s": if not validateOptionPair(currFlag, currValue): return False self.skipSteps = currValue elif currFlag == "-o": if not validateOptionPair(currFlag, currValue): return False self.overrideFile = currValue elif currFlag == "-d": if not validateOptionPair(currFlag, currValue): return False for definePair in currValue.split(","): splitPair = definePair.split("=") if len(splitPair) != 2: logger.writeError("Invalid define pair given, " + definePair) return False if splitPair[0].lower() in self.commandLineDefines: logger.writeError("Define pair already given, " + definePair) return False self.commandLineDefines[splitPair[0]] = splitPair[1] elif currFlag == "-n": if not validateOptionPair(currFlag, currValue): return False names = currValue.split(",") if len(names) == 0: logger.writeError("Build target names command-line option used but no names specified.") return False for name in names: currName = name.lower().strip() if currName == "": logger.writeError("Empty build target name given on command-line (option '-n')") return False self.targetsToBuild.append(currName) elif currFlag == "-g": if not validateOptionPair(currFlag, currValue): return False names = currValue.split(",") if len(names) == 0: logger.writeError("Override group command-line option used but no names specified.") return False for name in names: currName = name.lower().strip() if currName == "": logger.writeError("Empty override group name given on command-line (option '-g')") return False self.overrideGroupNames.append(currName) else: logger.writeError("Command-line argument '" + currArg + "' not understood") return False return True
def importTargets(options): logger.setLogger("console") finalTargets = [] ignoredTargets = [] partialImport = False fetchStep = commands.BuildStep("fetch", commands.__getFetchCommand(None)) unpackStep = commands.BuildStep("unpack", commands.__getUnpackCommand(None)) tempDir = tempfile.mkdtemp(prefix="mixdown-") options.downloadDir = os.path.join(tempDir, "mdDownloads") while len(options.targetsToImport) != 0: target = options.targetsToImport.pop(0) logger.writeMessage("Analyzing target...", target.name) logger.writeMessage("Extracting target...", target.name) target.outputPath = os.path.join(tempDir, target.name) target.currBuildStep = fetchStep if not commands.buildStepActor(target, options, None): utilityFunctions.removeDir(tempDir) return None, False target.currBuildStep = unpackStep if not commands.buildStepActor(target, options, None): utilityFunctions.removeDir(tempDir) return None, False #Generate build files and find possible dependencies possibleDeps = [] if cmake.isCMakeProject(target.path): logger.writeMessage("CMake project found...", target.name) logger.writeMessage("Analyzing for dependencies...", target.name) possibleDeps = cmake.getDependencies(target.path, target.name) elif autoTools.isAutoToolsProject(target.path): logger.writeMessage("Auto Tools project found...", target.name) if not os.path.exists(os.path.join(target.path, "configure")): if not autoTools.generateConfigureFiles( target.path, target.name): utilityFunctions.removeDir(tempDir) return None, False logger.writeMessage("Analyzing for dependencies...", target.name) possibleDeps = autoTools.getDependencies(target.path, target.name) if possibleDeps == None: target.comment = "Unable to parse 'configure --help' output. MixDown cannot determine dependencies for this target." logger.writeError(target.comment, target.name) partialImport = True possibleDeps = [] elif make.isMakeProject(target.path): target.comment = "Make project found. MixDown cannot determine dependencies from Make projects." logger.writeError(target.comment, target.name) partialImport = True else: target.comment = "Unknown build system found. MixDown cannot determine dependencies or build commands." logger.writeError(target.comment, target.name) partialImport = True #Find actual dependencies for possibleDependency in possibleDeps: if getTarget(possibleDependency, finalTargets + options.targetsToImport): logger.writeMessage( "Known dependency found (" + possibleDependency + ")", target.name) target.dependsOn.append(possibleDependency) continue elif options.interactive and possibleDependency in ignoredTargets: logger.writeMessage( "Previously ignored dependency found (" + possibleDependency + ")", target.name) continue if searchForPossibleAliasInList( possibleDependency, finalTargets + options.targetsToImport, options.interactive): target.dependsOn.append(possibleDependency) elif not options.interactive: logger.writeMessage( "Ignoring unknown dependency (" + possibleDependency + ")", target.name) else: logger.writeMessage( "Unknown dependency found (" + possibleDependency + ")", target.name) userInput = raw_input( possibleDependency + ": Input location, target name, or blank to ignore:" ).strip() if userInput == "": ignoredTargets.append(possibleDependency) elif os.path.isfile(userInput) or os.path.isdir( userInput) or utilityFunctions.isURL(userInput): name = target.targetPathToName(userInput) if name == "": return None, False newTarget = target.Target(name, userInput) options.targetsToImport.append(newTarget) if target.normalizeName( possibleDependency) != target.normalizeName( userInput): newTarget.aliases.append(possibleDependency) target.dependsOn.append(possibleDependency) else: aliasTarget = getTarget( userInput, finalTargets + options.targetsToImport, possibleDependency) if aliasTarget != None: logger.writeMessage("Alias added (" + userInput + ")", aliasTarget.name) target.dependsOn.append(possibleDependency) else: aliasLocation = raw_input( userInput + ": Target name not found in any known targets. Location of new target:" ).strip() if os.path.isfile(aliasLocation) or os.path.isdir( aliasLocation) or utilityFunctions.isURL( aliasLocation): name = target.targetPathToName(aliasLocation) if name == "": return None, False newTarget = target.Target(name, aliasLocation) notReviewedTargets.append(newTarget) if target.normalizeName(possibleDependency ) != target.normalizeName( aliasLocation): newTarget.aliases.append(possibleDependency) target.dependsOn.append(possibleDependency) else: logger.writeError( userInput + ": Alias location not understood.", exitProgram=True) finalTargets.append(target) #Create project for targets projects = project.Project("ProjectNameNotDetermined", finalTargets) if not projects.examine(options): logger.writeError("Project failed examination", exitProgram=True) if not projects.validate(options): logger.writeError("Project failed validation", exitProgram=True) mainTargetPath = projects.targets[0].origPath if utilityFunctions.isURL(mainTargetPath): mainTargetPath = utilityFunctions.URLToFilename(mainTargetPath) mainTargetName, mainTargetVersion = utilityFunctions.splitFileName( mainTargetPath) if mainTargetVersion != "": projects.name = mainTargetName + "-" + mainTargetVersion else: projects.name = mainTargetName projects.path = projects.name + ".md" for target in projects.targets: target.outputPath = "" if projects.examine(options): logger.writeMessage("\nFinal targets...\n\n" + str(projects)) projects.write() utilityFunctions.removeDir(tempDir) return projects, partialImport
def buildStepActor(target, options, project, lock=None): if target.isStepToBeSkipped(target.currBuildStep.name): try: if lock: lock.acquire() logger.reportSkipped(target.name, target.currBuildStep.name, "Target specified to skip step") finally: if lock: lock.release() return True if target.isStepPreviouslyDone(target.currBuildStep.name): try: if lock: lock.acquire() logger.reportSkipped( target.name, target.currBuildStep.name, "Build step successfully built in previous MixDown build" ) finally: if lock: lock.release() return True try: if lock: lock.acquire() logger.reportStart(target.name, target.currBuildStep.name) # Refresh defines before start of every step project.setTargetFieldsAsDefines(options.defines) finally: if lock: lock.release() returnCode = None timeStart = time.time() command = options.defines.expand(target.currBuildStep.command) isPythonCommand, namespace, function = python.parsePythonCommand(command) if isPythonCommand: success = python.callPythonCommand(namespace, function, target, options, project) if not success: returnCode = 1 else: returnCode = 0 else: try: if lock: lock.acquire() logger.writeMessage("Executing command: " + command, target.name, target.currBuildStep.name, True) finally: if lock: lock.release() if not os.path.exists(target.path): logger.writeError( target.name + "'s path does not exist when about to execute build command in step " + target.currBuildStep.name + ".", filePath=target.path, ) returnCode = 1 else: outFd = logger.getOutFd(target.name, target.currBuildStep.name) returnCode = utilityFunctions.executeSubProcess(command, target.path, outFd) timeFinished = time.time() timeElapsed = timeFinished - timeStart if returnCode != 0: target.currBuildStep.success = False try: if lock: lock.acquire() logger.reportFailure(target.name, target.currBuildStep.name, timeElapsed, returnCode) finally: if lock: lock.release() return False target.currBuildStep.success = True try: if lock: lock.acquire() logger.reportSuccess(target.name, target.currBuildStep.name, timeElapsed) finally: if lock: lock.release() return True
def readGroups(filename): if not os.path.exists(filename): logger.writeError("Given Override file path did not exist. Check your -o command-line option.", filePath=filename) return None groups = list() overrideGroup = None tokenizer = Tokenizer(filename) if not tokenizer.tokenize(): return None tokens = tokenizer.tokens i = 0 lengthOfTokens = len(tokens) while i < lengthOfTokens: #Syntax start: <name>[, <name>]* overrideGroup = OverrideGroup() while not (tokens[i].type == TokenType.Symbol and tokens[i].value == '{'): if len(overrideGroup.name) > 50: logger.writeError("Override group name exceeded limit (50).", filePath=filename) return None if tokens[i].type == TokenType.Identifier: overrideGroup.name.append(tokens[i].value.lower()) i += 1 if i >= lengthOfTokens: logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None if tokens[i].type == TokenType.Symbol and tokens[i].value == '{': #End of override group name syntax hit break elif tokens[i].type == TokenType.Symbol and tokens[i].value == ',': i += 1 if i >= lengthOfTokens: logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None else: logger.writeError("Expected ',' got '" + tokens[i].value + "'", filePath=filename) return None else: logger.writeError("Expected Override group name, got '" + tokens[i].value + "'", filePath=filename) return None #Syntax end #Syntax start: #{ # <Identifier> = <string> # ... # <Identifier> = <string> #} if not (tokens[i].type == TokenType.Symbol and tokens[i].value == '{'): logger.writeError("Expected '{' got '" + tokens[i].value + "'", filePath=filename) return None i += 1 if i >= lengthOfTokens: logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None while True: if tokens[i].type == TokenType.Symbol and tokens[i].value == '}': i += 1 break overrideName = "" overrideValue = "" #Syntax start: <Identifier> = <string> if not tokens[i].type == TokenType.Identifier: logger.writeError("Expected Override identifier got '" + tokens[i].value + "'", filePath=filename) return None overrideNameOriginal = tokens[i].value overrideName = tokens[i].value.lower() i += 1 if i >= lengthOfTokens: logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None if not (tokens[i].type == TokenType.Symbol and tokens[i].value == '='): logger.writeError("Expected '=' got '" + tokens[i].value + "'", filePath=filename) return None i += 1 if i >= lengthOfTokens: logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None if not tokens[i].type == TokenType.String: logger.writeError("Expected Override value string got '" + tokens[i].value + "'", filePath=filename) return None overrideString = tokens[i].value i += 1 if i >= lengthOfTokens: logger.writeError("Parsing ended inside of Override group. Please finish file and re-run MixDown.", filePath=filename) return None overrideGroup[overrideName] = overrideString #Syntax end #Syntax end overrideGroupNames = ", ".join(overrideGroup) for group in groups: if ", ".join(group.name) == overrideGroupNames: logger.writeError("Duplicate override group name found: " + overrideGroupNames, filePath=options.overrideFile) return None groups.append(overrideGroup) return groups