def _compileSrc(be, compileCmd, source, objPaths, buildStatus):
"""
be (in): BuildElements object
compileCmd (in): the compile command so far
source (in): the c or cpp source file to compile (every source file gets it's own object file)
objPaths (out): list of all object paths that will be passed to the linker
buildStatus (out): build status for this particular compile
"""
if not os.path.exists(source):
buildStatus.writeError(source + " is missing, exiting build")
return
objFile = os.path.basename(source)
objFile = objFile.replace(os.path.splitext(source)[1], be.objExt)
objPath = os.path.join(be.buildPath, objFile)
objPaths.append(objPath)
# check if it's up to date
objExisted = os.path.exists(objPath)
if objExisted:
objTimestamp = float(os.stat(objPath).st_mtime)
if not utils.sourceNeedsBuilding(be.incPaths, source, objTimestamp):
buildStatus.status = "up to date"
return
# Microsoft Visual C has to have the objPathFlag cuddled up directly next to the objPath - no space in between them (grrr)
if be.compiler.startswith("msvc"):
cmd = compileCmd + [source, be.objPathFlag + objPath]
else:
cmd = compileCmd + [source, be.objPathFlag, objPath]
if be.showCompilerCmds:
log.info("\n" + " ".join(cmd) + "\n")
buildStatus.description = utils.runCmd(cmd)
if os.path.exists(objPath):
if objExisted:
if float(os.stat(objPath).st_mtime) > objTimestamp:
buildStatus.status = "built"
else:
buildStatus.status = "built"
if buildStatus.status == "built":
buildStatus.description = "compiled " + os.path.basename(source)
def build(argv=[], buildingLib=False):
"""
the main function, does the heavy lifting
argv (in): command line arguments
"""
#
# cleaning
#
if "-cl" in argv:
return clean(argv)
if "-cla" in argv:
return cleanall(argv)
try:
be = BuildElements(argv)
except Exception as e:
log.info(e.args[0])
return False
buildStatus = BuildStatus(be.target, be.buildPath) # final build status
# lock - early return
if be.locked and os.path.exists(be.targetInstallPath):
buildStatus.writeInfo("locked", be.target + " is locked")
return True
#
# building
#
startTime = time.time()
threading = True
log.info("building " + be.infoStr)
if not os.path.exists(be.installPath):
utils.createDirs(be.installPath)
if not os.path.exists(be.buildPath):
os.makedirs(be.buildPath)
incPathList = []
for incPath in be.incPaths:
if os.path.exists(incPath):
incPathList += ["-I", incPath]
else:
log.warning("incPath {0} doesn't exist".format(incPath))
for (
extIncPath
) in (
be.extIncPaths
): # external include libs (for cases where 3rd party header includes are using "" instead of <> ie Unreal)
if os.path.exists(incPath):
incPathList += ["-I", extIncPath]
else:
log.warning("extIncPath {0} doesn't exist".format(extIncPath))
definesList = []
for define in be.defines:
definesList += ["-D", define]
#
# qt moc file compilation
#
mocPaths = []
for qtClass in be.qtClasses:
found = False
mocPath = "{0}/moc_{1}.cpp".format(be.buildPath, qtClass)
qtClassHeader = qtClass + ".h"
for (
incPath
) in be.incPaths: # find the header file, # TODO: should there be a separate list of headers ie be.mocIncPaths?
includePath = incPath + "/" + qtClassHeader
if not os.path.exists(includePath):
continue
if (
os.path.exists(mocPath)
and float(os.stat(mocPath).st_mtime) < float(os.stat(includePath).st_mtime)
or not os.path.exists(mocPath)
):
buildStatus.description = "qt moc: " + utils.runCmd(
["moc"] + definesList + [includePath, "-o", mocPath]
)
if not os.path.exists(mocPath):
buildStatus.writeError(buildStatus.description)
return False
mocPaths.append(mocPath)
found = True
if not found:
buildStatus.writeError("can't find {0} for qt moc compilation".format(qtClassHeader))
return False
for mocPath in mocPaths:
be.sources.append(mocPath)
buildStatusDeps = [] # the build status for each dependency: objs and libs
threads = []
i = 0
#
# compile
#
objPaths = []
cmd = [be.compilerCmd, be.objFlag] + incPathList + definesList + be.flags
if threading:
for source in be.sources:
buildStatusDep = BuildStatus(source)
buildStatusDeps.append(buildStatusDep)
thread = Thread(None, target=_compileSrc, args=(be, cmd, source, objPaths, buildStatusDep))
thread.start()
threads.append(thread)
i += 1
else:
for source in be.sources:
buildStatusDep = BuildStatus(source)
buildStatusDeps.append(buildStatusDep)
_compileSrc(be, cmd, source, objPaths, buildStatusDep)
i += 1
#
# build library dependencies
#
libCmds = []
libsBuilding = []
if be.binaryType == "exe" or be.binaryType == "plugin":
for lib in be.libs:
libName = lib
if be.compiler.startswith("msvc"):
libCmds += [
libName + be.staticExt
] # you need to link against the .lib stub file even if it's ultimately a .dll that gets linked
else:
libCmds += [be.libFlag, libName]
# check if the lib has a directory for building
if threading:
for libSrcDir in be.libSrcPaths:
libSrcDir = os.path.join(libSrcDir, lib)
if os.path.exists(libSrcDir):
libsBuilding.append(lib)
buildStatusDep = BuildStatus(lib)
buildStatusDeps.append(buildStatusDep)
thread = Thread(None, target=_buildLib, args=(be, libSrcDir, buildStatusDep))
thread.start()
threads.append(thread)
i += 1
break
else:
for libSrcPath in be.libSrcPaths:
if not os.path.exists("libSrcPath"):
log.warning("libSrcPath {0} doesn't exist".format(libSrcPath))
continue
libSrcPath = os.path.join(libSrcPath, lib)
if os.path.exists(libSrcPath):
libsBuilding.append(lib)
buildStatusDep = BuildStatus(lib)
buildStatusDeps.append(buildStatusDep)
_buildLib(be, libSrcDir, buildStatusDep)
i += 1
break
# wait for all the threads before checking the results
for thread in threads:
thread.join()
allUpToDate = True
for buildStatusDep in buildStatusDeps:
if buildStatusDep.status == "failed":
buildStatus.writeError(
"{0} failed because {1} failed because...\n\n{2}\n...determined in seconds\n\n".format(
be.infoStr,
buildStatusDep.name,
buildStatusDep.description.encode("ascii", "ignore"),
str(int(time.time() - startTime)),
)
)
return False
elif buildStatusDep.status == "built":
allUpToDate = False
# revise the library paths
for i in range(len(be.libPaths)):
revisedLibPath = be.libPaths[i] + be.binaryRelPath
if os.path.exists(revisedLibPath):
be.libPaths[i] = revisedLibPath
#
# linking
#
linkCmd = []
if allUpToDate and os.path.exists(be.targetInstallPath):
buildStatus.writeInfo(
"up to date",
"{0} is up to date, determined in {1} seconds\n".format(be.infoStr, str(int(time.time() - startTime))),
)
if not buildingLib:
_runPostScript()
return True
# microsoft's compiler / linker can only handle so many characters on the command line
msvcLinkCmdFilePath = be.buildPath + "/linkCmd"
if be.compiler.startswith("msvc"):
msvcLinkCmd = '{0}"{1}" {2} {3}'.format(
be.targetFlag, be.targetInstallPath, " ".join(objPaths), " ".join(libCmds)
)
msvcLinkCmdFp = open(msvcLinkCmdFilePath, "w")
msvcLinkCmdFp.write(msvcLinkCmd)
msvcLinkCmdFp.close()
linkCmd += [be.linker, "@" + msvcLinkCmdFilePath]
if be.showLinkerCmds:
log.info("\nmsvcLinkCmd: {0}\n".format(msvcLinkCmd))
else:
linkCmd += [be.linker, be.targetFlag, be.targetInstallPath] + objPaths + libCmds
if be.binaryType != "static": # TODO: is this the case for msvc?
linkCmd += be.linkFlags
if (
be.binaryType == "exe"
or be.binaryType == "plugin"
or (be.compilerRoot == "msvc" and be.binaryType == "dynamic")
):
for libPath in be.libPaths:
if not os.path.exists(libPath):
log.warning("libPath {0} doesn't exist".format(libPath))
continue
if be.compiler.startswith("msvc"):
linkCmd += [be.libPathFlag + os.path.normpath(libPath)]
else:
linkCmd += [be.libPathFlag, os.path.normpath(libPath)]
# get the timestamp of the existing target if it exists
linked = False
targetExisted = False
oldTargetTimeStamp = None
if os.path.exists(be.targetInstallPath):
oldTargetTimeStamp = float(os.stat(be.targetInstallPath).st_mtime)
targetExisted = True
if be.showLinkerCmds:
log.info("\n{0}\n".format(" ".join(linkCmd)))
buildStatus.description = utils.runCmd(linkCmd)
if os.path.exists(be.targetInstallPath):
if targetExisted:
if float(os.stat(be.targetInstallPath).st_mtime) > oldTargetTimeStamp:
linked = True
else:
linked = True
if linked:
log.info("linked " + be.infoStr)
else:
buildStatus.writeError("linking failed because " + buildStatus.description)
return False
# copy dynamic library dependencies (built by this build) to the install path
if be.binaryType == "exe" or be.binaryType == "plugin":
for lib in libsBuilding:
for libPath in be.libPaths:
dynamicPath = libPath + "/"
if be.compilerRoot == "gcc" or be.compilerRoot == "clang":
dynamicPath += "lib"
dynamicPath += lib + be.dynamicExt
if os.path.exists(dynamicPath):
utils.copyfile(dynamicPath, be.installPath)
buildStatus.writeInfo(
"built",
"{0} built {1}\ncompleted in {2} seconds\n".format(
be.infoStr, be.targetInstallPath, str(int(time.time() - startTime))
),
)
sys.stdout.flush()
# run a post-build script if it exists
if not buildingLib:
_runPostScript()
return True
def __init__(self, argv):
'''
argv (input): a list of arguments with a flag, value pairing ie ['-c', 'gcc'] (where -c is the flag and gcc is the value)
'''
# get any arguments passed in
if type(argv) is not list:
raise Exception('args must be a list, not a {0}'.format(argv))
self.target = ''
self.binaryType = '' # exe, static, dynamic, plugin
self.compiler = '' # g++-4.4 g++ clang++ msvc110 etc
self.osType = '' # linux, osx, windows
self.binaryFormat = '64bit' # 32bit, 64bit etc
self.buildType = 'debug' # debug, release etc
self.hiddenFiles = False # if pybythec files are "hidden files"
self.filetype = '' # elf, mach-o, pe
self.multithread = True
self.locked = False
self.showCompilerCmds = False
self.showLinkerCmds = False
self.buildDir = 'pybythec'
self.hideBuildDirs = False
self.installPath = '.' #self.buildDir
self.configKeys = [] # can be declared in the config files
self.customKeys = [] # custom keys that are both declared on the command line and found in the config file(s)
self.sources = []
self.libs = []
self.defines = []
self.flags = []
self.linkFlags = []
self.incPaths = []
self.extIncPaths = [] # these will not be checked for timestamps
self.libPaths = []
self.libSrcPaths = []
self.keys = []
self.qtClasses = []
self.libInstallPathAppend = True
self.plusplus = True
# defaults
if platform.system() == 'Linux':
if not len(self.osType):
self.osType = 'linux'
if not len(self.compiler):
self.compiler = 'g++'
if not len(self.filetype):
self.filetype = 'elf'
elif platform.system() == 'Darwin':
if not len(self.osType):
self.osType = 'osx'
if not len(self.compiler):
self.compiler = 'clang++'
if not len(self.filetype):
self.filetype = 'mach-o'
elif platform.system() == 'Windows':
if not len(self.osType):
self.osType = 'windows'
if not len(self.compiler):
i = 25 # NOTE: hopefully that covers enough VisualStudio releases
vcPath = 'C:/Program Files (x86)/Microsoft Visual Studio {0}.0/VC'
foundVc = False
while i > 5:
if os.path.exists(vcPath.format(i)):
foundVc = True
break
i -= 1
if foundVc:
self.compiler = 'msvc-{0:02}0'.format(i)
else:
raise Exception('can\'t find a compiler for Windows')
if not len(self.filetype):
self.filetype = 'pe'
else:
raise Exception('os does not appear to be Linux, OS X or Windows')
#
# parse the args
#
args = dict()
argKey = str()
keyFound = False
for arg in argv:
if keyFound:
args[argKey] = arg
keyFound = False
continue
if arg == '-cl' or arg == '-cla': # cleaning
args[arg] = ''
elif arg == '-c' or arg == '-os' or arg == '-b' or arg == '-bf' or arg == '-d' or arg == '-p' or arg == '-ck':
argKey = arg
keyFound = True
elif arg == '-v':
raise Exception('version: {0}'.format(utils.__version__))
else:
raise Exception('{0} is not a valid argumnet\nvalid arguments:\n\n'
'-c compiler: any variation of gcc, clang, or msvc ie g++-4.4, msvc110\n'
'-os operating system: currently linux, osx, or windows\n'
'-b build type: debug release etc \n'
'-bf binary format: 32bit, 64bit etc\n'
'-p path to a pybythec project config file (json format)\n'
'-cl clean the build\n'
'-cla clean the build and the builds of all library dependencies\n'
'-v version\n'
'-ck custom keys that you want this build to use (comma delineated, no spaces ie foo,bar)\n'
'-d directory of the library being built, likely only used when building a library as a dependency (ie from a project)\n'.format(arg))
self.cwDir = os.getcwd()
if '-d' in args:
self.cwDir = args['-d']
# json config files
globalCf = None
projectCf = None
localCf = None
# global config
if not globalCf and '-g' in args:
globalCf = utils.loadJsonFile(args['-g'])
if 'PYBYTHEC_GLOBALS' in os.environ:
globalCf = utils.loadJsonFile(os.environ['PYBYTHEC_GLOBALS'])
if not globalCf:
globalCf = utils.loadJsonFile('.pybythecGlobals.json')
if not globalCf:
globalCf = utils.loadJsonFile('pybythecGlobals.json')
if not globalCf:
homeDirPath = ''
if platform.system() == 'Windows':
homeDirPath = os.environ['USERPROFILE']
else:
homeDirPath = os.environ['HOME']
globalCf = utils.loadJsonFile(homeDirPath + '/.pybythecGlobals.json')
if not globalCf:
globalCf = utils.loadJsonFile(homeDirPath + '/pybythecGlobals.json')
if not globalCf:
log.warning('no global pybythec json file found')
# project config
if 'PYBYTHEC_PROJECT' in os.environ:
projectCf = os.environ['PYBYTHEC_PROJECT']
if not projectCf and '-p' in args:
projectCf = utils.loadJsonFile(args['-p'])
if not projectCf:
projectCf = utils.loadJsonFile(self.cwDir + '/pybythecProject.json')
if not projectCf:
projectCf = utils.loadJsonFile(self.cwDir + '/.pybythecProject.json')
# local config, expected to be in the current working directory
localConfigPath = self.cwDir + '/pybythec.json'
if not os.path.exists(localConfigPath):
localConfigPath = self.cwDir + '/.pybythec.json'
if os.path.exists(localConfigPath):
localCf = utils.loadJsonFile(localConfigPath)
if globalCf is not None:
self._getBuildElements(globalCf)
if projectCf is not None:
self._getBuildElements(projectCf)
if localCf is not None:
self._getBuildElements(localCf)
# command line overrides
if '-os' in args:
self.osType = args['-os']
if '-b' in args:
self.buildType = args['-b']
if '-bf' in args:
self.binaryFormat = args['-bf']
# compiler special case: os specific compiler selection
if type(self.compiler) == dict:
compiler = []
self.keys = [self.osType]
if globalCf and 'compiler' in globalCf:
self._getArgsList(compiler, globalCf['compiler'])
if projectCf and 'compiler' in projectCf:
self._getArgsList(compiler, projectCf['compiler'])
if localCf and 'compiler' in localCf:
self._getArgsList(compiler, localCf['compiler'])
if len(compiler):
self.compiler = compiler[0]
# one final commandline override: the compiler
if '-c' in args:
self.compiler = args['-c']
# TODO: verify things like does this compiler actually exist (to prevent getting poor error messages)
# currently compiler root can either be gcc, clang or msvc
self.compilerRoot = self.compiler
if self.compilerRoot.startswith('gcc') or self.compilerRoot.startswith('g++'):
self.compilerRoot = 'gcc'
elif self.compilerRoot.startswith('clang') or self.compilerRoot.startswith('clang++'):
self.compilerRoot = 'clang'
elif self.compilerRoot.startswith('msvc'):
self.compilerRoot = 'msvc'
else:
raise Exception('unrecognized compiler {0}'.format(self.compiler))
# compiler version
self.compilerVersion = ''
v = self.compiler.split('-')
if len(v) > 1:
self.compilerVersion = '-' + v[1]
self.keys = ['all', self.compilerRoot, self.compiler, self.osType, self.binaryType, self.buildType, self.binaryFormat]
# custom keys
if '-ck' in args:
cmdLineKeys = args['-ck']
for ck in self.configKeys:
if type(cmdLineKeys) == str:
if ck == cmdLineKeys:
self.customKeys.append(ck)
else: # assume list
if ck in cmdLineKeys:
self.customKeys.append(ck)
self.keys += self.customKeys
if self.multithread:
self.keys.append('multithread')
if globalCf is not None:
self._getBuildElements2(globalCf)
if projectCf is not None:
self._getBuildElements2(projectCf)
if localCf is not None:
self._getBuildElements2(localCf)
# deal breakers
if not len(self.target):
raise Exception('no target specified')
elif not len(self.binaryType):
raise Exception('no binary type specified')
elif not len(self.binaryFormat):
raise Exception('no binary format specified')
elif not len(self.buildType):
raise Exception('no build type specified')
elif not len(self.sources):
raise Exception('no source files specified')
if not (self.binaryType == 'exe' or self.binaryType == 'static' or self.binaryType == 'dynamic' or self.binaryType == 'plugin'):
raise Exception('unrecognized binary type: ' + self.binaryType)
if self.hideBuildDirs:
self.buildDir = '.' + self.buildDir
#
# compiler config
#
self.compilerCmd = self.compiler
self.linker = ''
self.targetFlag = ''
self.libFlag = ''
self.libPathFlag = ''
self.objExt = ''
self.objPathFlag = ''
self.staticExt = ''
self.dynamicExt = ''
self.pluginExt = ''
#
# gcc / clang
#
if self.compilerRoot == 'gcc' or self.compilerRoot == 'clang':
if not self.plusplus: # if forcing plain old C (ie when a library is being built as a dependency that is only C compatible)
if self.compilerRoot == 'gcc':
self.compilerCmd = self.compilerCmd.replace('g++', 'gcc')
elif self.compilerRoot == 'clang':
self.compilerCmd = self.compilerCmd.replace('clang++', 'clang')
self.objFlag = '-c'
self.objExt = '.o'
self.objPathFlag = '-o'
self.defines.append('_' + self.binaryFormat.upper()) # TODO: you sure this is universal?
# link
self.linker = self.compilerCmd # 'ld'
self.targetFlag = '-o'
self.libFlag = '-l'
self.libPathFlag = '-L'
self.staticExt = '.a'
self.dynamicExt = '.so'
self.pluginExt = '.so'
# log.info('*** filetype {0}'.format(self.filetype))
if self.filetype == 'mach-o':
self.dynamicExt = '.dylib'
self.pluginExt = '.bundle'
if self.binaryType == 'static' or self.binaryType == 'dynamic':
self.target = 'lib' + self.target
if self.binaryType == 'exe':
pass
elif self.binaryType == 'static':
self.target = self.target + '.a'
self.linker = 'ar'
self.targetFlag = 'r'
elif self.binaryType == 'dynamic':
self.target = self.target + self.dynamicExt
elif self.binaryType == 'plugin':
self.target = self.target + self.pluginExt
#
# msvc / msvc
#
elif self.compilerRoot == 'msvc':
# compile
self.compilerCmd = 'cl'
self.objFlag = '/c'
self.objExt = '.obj'
self.objPathFlag = '/Fo'
# link
self.linker = 'link'
self.targetFlag = '/OUT:'
self.libFlag = ''
self.libPathFlag = '/LIBPATH:'
self.staticExt = '.lib'
self.dynamicExt = '.dll'
if self.binaryFormat == '64bit':
self.linkFlags.append('/MACHINE:X64')
if self.binaryType == 'exe':
self.target += '.exe'
elif self.binaryType == 'static':
self.target += self.staticExt
self.linker = 'lib'
elif self.binaryType == 'dynamic' or self.binaryType == 'plugin':
self.target += self.dynamicExt
self.linkFlags.append('/DLL')
# make sure the compiler is in PATH
if utils.runCmd(self.compilerCmd).startswith('[WinError 2]'):
raise Exception('compiler not found, check the paths set in bins')
else:
raise Exception('unrecognized compiler root: ' + self.compilerRoot)
#
# determine paths
#
self.installPath = utils.makePathAbsolute(self.cwDir, self.installPath)
self._resolvePaths(self.cwDir, self.sources)
self._resolvePaths(self.cwDir, self.incPaths)
self._resolvePaths(self.cwDir, self.extIncPaths)
self._resolvePaths(self.cwDir, self.libPaths)
self._resolvePaths(self.cwDir, self.libSrcPaths)
self.binaryRelPath = '/{0}/{1}/{2}'.format(self.buildType, self.compiler, self.binaryFormat)
binRelPath = self.binaryRelPath
for ck in self.customKeys:
binRelPath += '/' + ck
self.buildPath = utils.makePathAbsolute(self.cwDir, './' + self.buildDir + binRelPath)
if self.libInstallPathAppend and (self.binaryType == 'static' or self.binaryType == 'dynamic'):
self.installPath += binRelPath
self.targetInstallPath = os.path.join(self.installPath, self.target)
self.infoStr = '{0} ({1} {2} {3}'.format(self.target, self.buildType, self.compiler, self.binaryFormat)
if len(self.customKeys):
for ck in self.customKeys:
self.infoStr += ' ' + ck
self.infoStr += ')'
