def __init__(self,
target,
command=None,
dependencies=[],
cwd=None,
redirectStdOutToTarget=False,
env=None,
stdout=None,
stderr=None,
commands=None):
BaseTarget.__init__(self, target, dependencies)
assert not (command
and commands), 'Cannot specify both command= and commands='
self.command = command
self.commands = commands
self.cwd = cwd
self.deps = PathSet(dependencies)
self.redirectStdOutToTarget = redirectStdOutToTarget
if redirectStdOutToTarget and isDirPath(target):
raise BuildException(
'Cannot set redirectStdOutToTarget and specify a directory for the target name - please specify a file instead: %s'
% target)
self.env = env
self.stdout, self.stderr = stdout, stderr
if stdout and redirectStdOutToTarget:
raise BuildException(
'Cannot set both redirectStdOutToTarget and stdout')
def __init__(self,
output,
compile,
main=None,
libs=None,
flags=None,
dependencies=None,
resources=None):
"""
@param output: the resulting .exe or .dll
@param compile: the input PathSet, path or list of .cs file(s)
@param main: The main class to execute if an exe is to be built.
If this is set then an executable will be created.
Otherwise this target will build a library.
@param libs: a list of input libraries (or a PathSet)
"""
self.compile = FilteredPathSet(_isDotNetFile, PathSet(compile))
self.main = main
self.flags = flags or []
self.libs = PathSet(libs or [])
self.resources = resources or []
BaseTarget.__init__(self, output, [
self.compile, self.libs, [x for (x, y) in self.resources],
dependencies or []
])
self.tags('c#')
def __init__(self, dest, archives):
"""
@param dest: the output directory (ending with a "/"). Never
specify a dest directory that is also written to by another
target (e.g. do not specify a build 'output' directory here).
@param archives: the input archives to be unpacked, which may be any
combination of strings, PathSets, FilteredArchiveContents and lists of these.
If these PathSets include mapping information, this
will be used to define where (under the dest directory) each
file from within that archive is copied (but cannot be used to
change the archive-relative path of each item).
For advanced cases, FilteredArchiveContents can be used to provide
customized mapping and filtering of the archive contents,
including manipulation of the destinations encoded within the
archive itself.
"""
if not dest.endswith('/'): raise BuildException('Unpack target destination must be a directory (ending with "/"), not: "%s"'%dest)
# NB: we could also support copying in non-archived files into the directory future too
# we should preserve the specified order of archives since it may
# affect what happens when they contain the same files and must
# overwrite each other
archives = [a if (isinstance(a, BasePathSet) or isinstance(a, FilteredArchiveContents)) else PathSet(a) for a in flatten(archives)]
BaseTarget.__init__(self, dest, [
(a.getDependency() if isinstance(a, FilteredArchiveContents) else a)
for a in archives])
self.archives = archives
def __init__(self, name, time, dependencies=None):
"""
name: the output filename
time: the length of time to sleep (in seconds)
"""
BaseTarget.__init__(self, name, dependencies or [])
self.time = float(time)
def __init__(self, archive, inputs): """ @param archive: The archive to be created, ending with ``.zip``. @param inputs: The files (usually pathsets) to be included in the archive. """ self.inputs = PathSet(inputs) BaseTarget.__init__(self, archive, self.inputs)
def __init__(self,
name,
getContents,
dependencies=None,
mode=None,
executable=False,
encoding=None,
args=None,
kwargs=None):
"""
Example usage::
WriteFile('${OUTPUT_DIR}/foo.txt', lambda context: '\\n'.join(['Foo:', context.expandPropertyValues('${FOO}')]))
@param name: the output filename
@param getContents:
a unicode character string (which will be subject to expansion),
or binary bytes, or a function that accepts a context as input
followed optionally by any specified 'args') and returns
the string/bytes that should be written to the file, using ``\\n`` for newlines
(not ``os.linesep`` - any occurrences of the newline character ``\\n`` in
the provided string will be replaced automatically with the
OS-specific line separator unless bytes are provided).
The function will be evaluated during the dependency resolution
phase (before the build phase commences).
@param mode: unix permissions to set with chmod on the destination files.
If not specified, default mode is used.
Ignored on Windows platforms.
@param executable: set to True to add Unix executable permissions (simpler
alternative to setting using mode)
@param encoding: The encoding to use for converting the str to bytes;
if not specified the ``common.fileEncodingDecider`` option is used.
@param args: optional tuple containing arguments that should be passed to
the getContents function, after the context argument (first arg).
@param kwargs: optional dictionary containing kwargs that should be passed
to the getContents function.
@param dependencies: any targets which need to be built in order to run this
target.
"""
BaseTarget.__init__(self, name, dependencies or [])
self.getContents = getContents
self.__args = args or ()
self.__kwargs = kwargs or {}
self.__resolved = None
self.__mode = mode
self.__executable = executable
self.__encoding = encoding
self.addHashableImplicitInputOption('common.fileEncodingDecider')
def __init__(self, bin, objects):
"""
@param bin: the output library
@param objects: a (list of) input objects
"""
self.objects = PathSet(objects)
BaseTarget.__init__(self, bin, self.objects)
self.tags('native')
def internal_clean(self, context): """ Calls the BaseTarget clean, not the target-specific clean """ try: deleteFile(self.__implicitInputsFile) except Exception: time.sleep(10.0) deleteFile(self.__implicitInputsFile) BaseTarget.clean(self.target, context)
def __init__(self, jar, compile, classpath, manifest, options=None, package=None, preserveManifestFormatting=False):
self.compile = FilteredPathSet(_isJavaFile, PathSet(compile)) if compile else None
self.classpath = PathSet(classpath)
self.package = PathSet(package)
self.manifest = manifest
BaseTarget.__init__(self, jar, [self.compile,self.classpath,self.package,
manifest if isinstance(manifest, str) else None])
for k,v in (options or {}).items(): self.option(k, v)
self.preserveManifestFormatting = preserveManifestFormatting
def __init__(self, destdir, source, classpath, options):
"""
@param destdir: directory to create docs in
@param source: a set of files to build from
@param classpath: a list of jars needed for the classpath
@param options: [DEPRECATED - use .option() instead]
"""
self.sources = PathSet(source)
self.classpath = PathSet(classpath)
BaseTarget.__init__(self, destdir, [self.sources, self.classpath])
for k,v in (options or {}).items(): self.option(k, v)
def __init__(self,
object,
source,
includes=None,
flags=None,
dependencies=None,
options=None):
"""
@param object: the object file to generate; see L{objectname}.
@param source: a (list of) source files
@param includes: a (list of) include directories, as strings or PathSets,
each with a trailing slash; the directories in the `native.include`
option are also added.
If this target depends on some include files that are generated by another target,
make sure it's a directory target since all include directories must either
exist before the build starts or be targets themselves.
If specifying a subdirectory of a generated directory, do this using DirGeneratedByTarget.
If you have a composite generated directory made up of several
file targets, wrap them in TargetsWithinDir before passing as the includes parameter.
@param flags: a list of compiler flags in addition to those in the
`native.cxx.flags`/`native.c.flags` option.
@param dependencies: a list of additional dependencies that need to be built
before this target. Usually this is not needed.
@param options: DEPRECATED; use .option() instead
"""
self.source = PathSet(source)
# currently we don't bother adding the native include dirs here as they're probably always going to be there
# for time being, explicitly cope with missing slashes, though really build authors should avoid this
self.includes = _AddTrailingDirectorySlashesPathSet(PathSet(includes))
self.flags = flatten([flags]) or []
# nb: do not include any individual header files in main target deps even if we've already
# got cached makedepends from a previous build,
# because it's possible they are no longer needed and no longer exist (and we don't want spurious
# build failures); this also has the advantage that it doesn't enlarge and slow down the stat cache
# during dep resolution of non-native targets, since it'll only be populated once we're into
# the up to date checking phase
BaseTarget.__init__(self, object,
PathSet([dependencies, source, self.includes]))
for k, v in (options or {}).items():
self.option(k, v)
self.tags('native')
def __init__(self, target, deps, fn, cleanfn=None):
"""
@param target: The target file/directory that will be built
@param deps: The list of dependencies of this target (paths, pathsets or lists)
@param fn: The functor used to build this target
@param cleanfn: The functor used to clean this target (optional, defaults to removing
the target file/dir)
"""
BaseTarget.__init__(self, target, deps)
self.fn = fn
self.cleanfn = cleanfn
self.deps = PathSet(deps)
def __init__(self, dest, src, relative=True):
"""
dest: the link to be created
src: what it points at
"""
if isinstance(dest, str) and dest.endswith('/'): raise BuildException('SymLink target can only be used for files, not directories') # for now
if not hasattr(os, 'symlink'): raise BuildException('SymLink target is not supported on this platform')
self.src = PathSet(src)
self.relative=relative
BaseTarget.__init__(self, dest, [self.src])
# technically we don't need to depend on the contents of src at all,
# but in practice it might be useful to have other targets depending on the
# link in which case it's a good idea to recreate the link when the
# thing it points to is rebuild - and it's a very quick operation anyway
self.tags('native')
def __init__(self, output, compile, classpath, options=None): """ @param output: output dir for class files @param compile: PathSet (or list) of things to compile @param classpath: PathSet (or list) of things to be on the classpath @param options: [DEPRECATED - use .option() instead] """ self.compile = FilteredPathSet(_isJavaFile, PathSet(compile)) self.classpath = PathSet(classpath) BaseTarget.__init__(self, output, [self.compile,self.classpath]) if options is not None: for k,v in options.items(): self.option(k, v)
def clean(self, context):
BaseTarget.clean(self, context)
args = [self.getOption('docker.path')]
environs = {
'DOCKER_HOST': self.getOption('docker.host')
} if self.getOption('docker.host') else {}
args.extend(['rmi', context.expandPropertyValues(self.imagename)])
# Allow warning/info logs but do not let failure of RMI cleanup break the build
rmiOptions = dict(self.options)
rmiOptions[ProcessOutputHandler.Options.ignoreReturnCode] = True
rmiOptions[
ProcessOutputHandler.Options.downgradeErrorsToWarnings] = True
call(args,
outputHandler=self.getOption('docker.outputHandlerFactory')(
'docker-rmi', treatStdErrAsErrors=False, options=rmiOptions),
timeout=self.getOption('process.timeout'),
env=environs)
def __init__(self, dest, src, implicitDependencies=None):
"""
@param dest: the output directory (ending with a "/") or file. Never
specify a dest directory that is also written to by another
target (e.g. do not specify an output directory here). If you need
to write multiple files to a directory, use separate Copy
targets for each, with file (rather than directory) target dest
names.
@param src: the input, which may be any combination of strings, PathSets and
lists of these. If these PathSets include mapping information, this
will be used to define where (under the dest directory) each
file is copied.
Note that only src files will be copied, any directory in the
src list will be created but its contents will not be copied
across - the only way to copy a directory is to use a FindPaths
(or FindPaths(DirGeneratedByTarget('...')))
for the src, which has the ability to find its contents on disk
(this is necessary to prevent complex race conditions and
build errors arising from implicit directory walking during
the execution phase - if all dir walking happens during
dependency resolution then such errors can be easily detected
before they cause a problem).
To create new empty directories that are not present in the source (mkdir),
you can use this simple trick which utilizes the fact that the current
directory ``.`` definitely exists. It doesn't copy anything from inside
(just copies only its 'existence') and uses a SingletonDestRenameMapper PathSet
to provide the destination::
SingletonDestRenameMapper('my-new-dest-directory/', './'),
@param implicitDependencies: provides a way to add additional implicit
dependencies that will not be part of src but may affect the
copy process (e.g. filtering in); this is intended for
use by subclasses, do not set this explicitly.
"""
src = PathSet(src)
BaseTarget.__init__(self, dest, [src, implicitDependencies])
self.src = src
self.mode = None # not yet supported, but may be if it turns out to be useful
self.addHashableImplicitInputOption('Copy.symlinks')
def __init__(self, output, jars, keystore, alias=None, storepass=None, manifestDefaults=None): """ @param output: The output directory in which to put the signed jars @param jars: The list (or PathSet) of input jars to copy and sign @param keystore: The path to the keystore @param alias: The alias for the keystore (optional) @param storepass: The password for the store file (optional) @param manifestDefaults: a dictionary of manifest entries to add to the existing manifest.mf file of each jar before signing. Entries in this dictionary will be ignored if the same entry is found in the original manifest.mf file already. """ self.jars = PathSet(jars) self.keystore = keystore self.alias = alias self.storepass = storepass self.manifestDefaults = manifestDefaults BaseTarget.__init__(self, output, [self.jars, self.keystore])
def __init__(self,
bin,
objects,
libs=None,
libpaths=None,
shared=False,
options=None,
flags=None,
dependencies=None):
"""
@param bin: the output binary. See L{exename}, L{libname}, L{staticlibname}.
@param objects: a (list of) input object
@param libs: a (list of) libraries linked against (optional) in platform-neutral format.
Can include list properties like '${FOO_LIB_NAMES[]}'.
@param libpaths: a (list of) additional library search directories (optional)
@param shared: if true compiles to a shared object (.dll or .so) (optional, defaults to false)
@param flags: a list of additional linker flags
@param options: [DEPRECATED - use .option() instead]
@param dependencies: a list of additional dependencies (targets or files)
"""
self.objects = PathSet(objects)
self.libs = libs or []
self.libpaths = PathSet(libpaths or [])
self.shared = shared
self.flags = flags or []
BaseTarget.__init__(self, bin,
PathSet(self.objects, (dependencies or [])))
for k, v in (options or {}).items():
self.option(k, v)
self.tags('native')
def __init__(self,
imagename,
inputs,
depimage=None,
dockerfile=None,
buildArgs=None,
dockerArgs=None):
"""
imagename: the name/tag of the image to build
"""
self.imagename = imagename
self.depimage = depimage
self.dockerfile = dockerfile
self.buildArgs = buildArgs
self.dockerArgs = dockerArgs
self.stampfile = '${BUILD_WORK_DIR}/targets/docker/.%s' % self.imageNameToFileName(
imagename)
self.depstampfile = '${BUILD_WORK_DIR}/targets/docker/.%s' % self.imageNameToFileName(
depimage) if depimage else None
self.inputs = PathSet(inputs)
BaseTarget.__init__(
self, self.stampfile,
inputs + ([self.depstampfile] if self.depstampfile else []))
def __init__(self, path):
"""
@param path: the output filename.
"""
BaseTarget.__init__(self, path, [])
def clean(self, context):
deleteFile(self._getMakeDependsFile(context))
BaseTarget.clean(self, context)
def main(args):
""" Command line argument parser.
"""
try:
usage = [
###############################################################################
'',
'eXtensible Python-based Build System %s on Python %s.%s.%s'% (XPYBUILD_VERSION, sys.version_info[0], sys.version_info[1], sys.version_info[2]),
'',
'xpybuild.py [operation]? [options]* [property=value]* [-x] [target|tag|regex]* ',
'',
'A regex containing * can be used instead of a target, but only if it uniquely ',
'identifies a single target. ',
'',
'Special pseudo-tags:',
' full Include all targets for a full build (the default)',
'',
'Special properties:',
' OUTPUT_DIR=output The main directory output will be written to',
' BUILD_MODE=release Specifies release, debug (or user-defined) mode',
' BUILD_NUMBER=n Build number string, for reporting and use by build',
'',
'Operations: ',
###############################################################################
' (if none is specified, the default operation is a normal build)',
' --clean Clean specified targets incl all deps',
' --rebuild Clean specified targets incl all deps then build',
' --rebuild-ignore-deps Clean only the specified targets (not deps) then ',
' (or --rid) build those targets and any missing dependencies, ',
' but not any out-of-date dependencies. This is a ',
' fast but less correct way to get a quick ',
' incremental build, so use with care. ',
'',
' -s --search <str> Show info on targets/tags/properties/options ',
' containing the specified substring or regex',
# hide these from usage (though they still work), as superceded by the more useful "-s" option
#' --ft --find-targets <str> List targets containing the specified substring',
#' --ti --target-info <str> Print details including build file location for ',
#' targets containing the specified substring',
' --targets List available targets and tags (filtered by any ',
' target or tag names specified on the command line)',
' --properties List properties that can be set and their ',
' defaults in this build file',
' --options List the target options available to build rules ',
' and their default values in this build file',
'',
'Options:',
' -x --exclude <target> Specifies a target or tag to exclude (unless ',
' needed as a dependency of an included target) ',
'',
' -J --parallel Build in parallel (this is the default). ',
' The number of workers is determined from the ',
' `build.workers` build file option or else the ',
' number of CPUs and the XPYBUILD_WORKERS_PER_CPU ',
' environment varable (default is currently 1.0), ',
' with an upper limit for this machine from the ',
' XPYBUILD_MAX_WORKERS variable. ',
' -j --workers <number> Override the number of worker threads to use for ',
' building. Use -j1 for single-threaded. ',
' (ignores any environment variables)',
'',
' -k --keep-going Continue rather than aborting on errors',
'',
' -n --dry-run Don\'t actually build anything, just print',
' what would be done (finds missing dependencies)',
'',
' --id --ignore-deps Skip all dependency/up-to-date checking: only ',
' clean/build targets that do not exist at all ',
' (faster builds, but no guarantee of correctness)',
'',
' -f --buildfile <file> Specify the root build file to import ',
' (default is ./root.xpybuild.py)',
'',
' -l --log-level LEVEL Set the log level to debug/info/critical',
' -L --logfile <file> Set the log file location',
' --timefile <file> Dump the time for each target in <file> at the',
' end of the run',
' --depgraph <file> Just resolve dependencies and dump them to <file>',
' --cpu-stats Log CPU utilisation stats',
' --random-priority Randomizes build order',
' --verify Performs additional verifications during the ',
' build to to help detect bugs in the build files. ',
' [verify is currently an experimental feature]',
' --profile Profiles all the worker threads',
' -F --format Message output format.',
' Options:',
] + [
' - '+ h for h in _registeredConsoleFormatters
] + [
]
if reduce(max, list(map(len, usage))) > 80:
raise Exception('Invalid usage string - all lines must be less than 80 characters')
# set up defaults
properties = {}
buildOptions = { "keep-going":False, "workers":0, "dry-run":False,
"ignore-deps":False, "logCPUUtilisation":False, "profile":False, "verify":False }
includedTargets = []
excludedTargets = []
task = _TASK_BUILD
buildFile = os.path.abspath('root.xpybuild.py')
logLevel = None
logFile = None
findTargetsPattern = None
format = "default"
opts,targets = getopt.gnu_getopt(args, "knJh?x:j:l:L:f:F:s:",
["help","exclude=","parallel","workers=","keep-going",
"log-level=","logfile=","buildfile=", "dry-run",
"targets", 'target-info=', 'ti=', "properties", "options", "clean", "rebuild", "rebuild-ignore-deps", "rid", "ignore-deps", "id",
"format=", "timefile=", "ft=", "find-targets=", "search=", "depgraph=", 'cpu-stats', 'random-priority', 'profile', 'verify'])
for o, a in opts: # option arguments
o = o.strip('-')
if o in ["?", "h", "help"]:
print('\n'.join(usage))
return 0
elif o in ["x", "exclude"]:
excludedTargets.append(a)
elif o in ["f", "buildfile"]:
buildFile = os.path.abspath(a)
elif o in ['targets']:
task = _TASK_LIST_TARGETS
elif o in ['find-targets', 'ft']:
task = _TASK_LIST_FIND_TARGETS
findTargetsPattern = BaseTarget._normalizeTargetName(a)
elif o in ['target-info', 'ti']:
task = _TASK_LIST_TARGET_INFO
findTargetsPattern = BaseTarget._normalizeTargetName(a)
elif o in ['search', 's']:
task = _TASK_LIST_SEARCH
searchPattern = BaseTarget._normalizeTargetName(a)
elif o in ['properties']:
task = _TASK_LIST_PROPERTIES
elif o in ['options']:
task = _TASK_LIST_OPTIONS
elif o in ['J', 'parallel']:
buildOptions['workers'] = 0
elif o in ['j', 'workers']:
buildOptions['workers'] = int(a)
elif o in ['l', 'log-level']:
logLevel = getattr(logging, a.upper(), None)
elif o in ['cpu-stats']:
buildOptions["logCPUUtilisation"] = True
elif o in ['random-priority']:
buildOptions["randomizePriorities"] = True
elif o in ['L', 'logfile']:
logFile = a
elif o in ['F', 'format']:
format = None
if a =='xpybuild': a = 'default' # for compatibility
for h in _registeredConsoleFormatters:
if h.upper() == a.upper():
format = h
if not format:
print('invalid format "%s"; valid formatters are: %s'%(a, ', '.join(_registeredConsoleFormatters.keys())))
print('\n'.join(usage))
return 1
elif o in ['clean']:
task = _TASK_CLEAN
buildOptions['keep-going'] = True
elif o in ['rebuild']:
task = _TASK_REBUILD
elif o in ['rebuild-ignore-deps', 'rid']:
task = _TASK_REBUILD
buildOptions['ignore-deps'] = True
elif o in ['id', 'ignore-deps']:
buildOptions['ignore-deps'] = True
elif o in ['k', 'keep-going']:
buildOptions['keep-going'] = True
elif o in ['n', 'dry-run']:
buildOptions['dry-run'] = True
elif o in ['timefile']:
buildOptions['timeFile'] = a
elif o in ['verify']:
buildOptions['verify'] = True
elif o in ['profile']:
buildOptions['profile'] = True
elif o in ['depgraph']:
buildOptions['depGraphFile'] = a
else:
assert False, "unhandled option: '%s'" % o
for o in targets: # non-option arguments (i.e. no -- prefix)
arg = o.strip()
if arg:
if '=' in arg:
properties[arg.split('=')[0].upper()] = arg.split('=')[1]
elif arg=='all': # pre-4.0 alias for all
includedTargets.append('full')
else:
includedTargets.append(BaseTarget._normalizeTargetName(arg))
# default is all
if (not includedTargets) or includedTargets==['']:
includedTargets = ['full']
except getopt.error as msg:
print(msg)
print("For help use --help")
return 2
threading.currentThread().setName('main')
logging.getLogger().setLevel(logLevel or logging.INFO)
if buildOptions["workers"] < 0: buildOptions["workers"] = 0 # means there's no override
outputBufferingDisabled = buildOptions['workers']==1 # nb: this also affects the .log handler below
# nb: it's possible workers=0 (auto) and will later be set to 1 but doesn't really matter much
# initialize logging to stdout - minimal output to avoid clutter, but indicate progress
hdlr = _registeredConsoleFormatters.get(format, None)
assert hdlr # shouldn't happen
wrapper = OutputBufferingStreamWrapper(sys.stdout, bufferingDisabled=outputBufferingDisabled)
# actually instantiate it
hdlr = hdlr(
wrapper,
buildOptions=buildOptions)
# Handler can override wrapper with a preference in either direction
if hdlr.bufferingDisabled: wrapper.bufferingDisabled = True
if hdlr.bufferingRequired: wrapper.bufferingDisabled = False
hdlr.setLevel(logLevel or logging.WARNING)
logging.getLogger().addHandler(hdlr)
log.info('Build options: %s'%{k:buildOptions[k] for k in buildOptions if k != 'workers'})
stdout = sys.stdout
# redirect to None, to prevent any target code from doing 'print' statements - should always use the logger
sys.stdout = None
listen_for_stack_signal() # make USR1 print a python stack trace
allTargets = ('full' in includedTargets) and not excludedTargets
try:
def loadBuildFile():
init = BuildInitializationContext(properties)
isRealBuild = (task in [_TASK_BUILD, _TASK_CLEAN, _TASK_REBUILD])
init._defineOption("process.timeout", 600)
init._defineOption("build.keepGoing", buildOptions["keep-going"])
# 0 means default behaviour
init._defineOption("build.workers", 0)
init.initializeFromBuildFile(buildFile, isRealBuild=isRealBuild)
# now handle setting real value of workers, starting with value from build file
workers = int(init._globalOptions.get("build.workers", 0))
# default value if not specified in build file
if workers <= 0:
workers = multiprocessing.cpu_count()
if os.getenv('XPYBUILD_WORKERS_PER_CPU'):
workers = min(workers, int(round(multiprocessing.cpu_count() * float(os.getenv('XPYBUILD_WORKERS_PER_CPU')))))
# machine/user-specific env var can cap it
if os.getenv('XPYBUILD_MAX_WORKERS'):
workers = min(workers, int(os.getenv('XPYBUILD_MAX_WORKERS')))
# finally an explicit command line --workers take precedence
if buildOptions['workers']: workers = buildOptions['workers']
if workers < 1: workers = 1
# finally write the final number of workers where it's available to both scheduler and targets
buildOptions['workers'] = workers
init._globalOptions['build.workers'] = workers
init._finalizeGlobalOptions()
return init
if buildOptions['profile']:
import cProfile, pstats
profiler = cProfile.Profile()
profiler.enable()
init = loadBuildFile()
# nb: don't import any modules that might define options (including outputhandler)
# until build file is loaded
# or we may not have a build context in place yet#
from xpybuild.internal.scheduler import BuildScheduler, logTargetTimes
if buildOptions['profile']:
profilepath = 'xpybuild-profile-%s.txt'%'parsing'
with open(profilepath, 'w') as f:
p = pstats.Stats(profiler, stream=f)
p.sort_stats('cumtime').print_stats(f)
p.dump_stats(profilepath.replace('.txt', '')) # also in binary format
log.critical('=== Wrote Python profiling output to: %s', profilepath)
def lookupTarget(s):
tfound = init.targets().get(s,None)
if not tfound and '*' in s:
matchregex = s.rstrip('$')+'$'
try:
matchregex = re.compile(matchregex, re.IGNORECASE)
except Exception as e:
raise BuildException('Invalid target regular expression "%s": %s'%(matchregex, e))
matches = [t for t in init.targets().values() if matchregex.match(t.name)]
if len(matches) > 1:
print('Found multiple targets matching pattern %s:'%(s), file=stdout)
print(file=stdout)
for m in matches:
print(m.name, file=stdout)
print(file=stdout)
raise BuildException('Target regex must uniquely identify a single target: %s (use tags to specify multiple related targets)'%s)
if matches: return matches[0]
if not tfound: raise BuildException('Unknown target name, target regex or tag name: %s'%s)
return tfound
# expand tags to targets here, and do include/exclude calculations
selectedTargets = set() # contains BaseTarget objects
for t in includedTargets:
tlist = init.tags().get(t,None)
if tlist:
selectedTargets.update(tlist)
else:
selectedTargets.add(lookupTarget(t))
for t in excludedTargets:
tlist = init.tags().get(t,None)
if tlist:
selectedTargets.difference_update(tlist)
else:
selectedTargets.discard(lookupTarget(t))
# convert findTargetsPattern to list
if findTargetsPattern:
findTargetsPattern = findTargetsPattern.lower()
# sort matches at start of path first, then anywhere in name, finally anywhere in type
# make 'all'/'full' into a special case that maps to all *selected* targets
# (could be different to 'all' tag if extra args were specified, but this is unlikely and kindof useful)
findTargetsList = [t for t in sorted(
init.targets().values() if allTargets else selectedTargets, key=lambda t:(
'/'+findTargetsPattern.lower() not in t.name.lower(),
findTargetsPattern.lower() not in t.name.lower(),
findTargetsPattern.lower() not in t.type.lower(),
t.name
)) if findTargetsPattern in t.name.lower() or findTargetsPattern in t.type.lower()
or findTargetsPattern == 'full' or findTargetsPattern == 'all']
if task == _TASK_LIST_PROPERTIES:
p = init.getProperties()
print("Properties: ", file=stdout)
pad = max(list(map(len, p.keys())))
if pad > 30: pad = 0
for k in sorted(p.keys()):
print(('%'+str(pad)+'s = %s') % (k, p[k]), file=stdout)
elif task == _TASK_LIST_OPTIONS:
options = init.mergeOptions(None)
pad = max(list(map(len, options.keys())))
if pad > 30: pad = 0
for k in sorted(options.keys()):
print(("%"+str(pad)+"s = %s") % (k, options[k]), file=stdout)
elif task == _TASK_LIST_TARGETS:
if len(init.targets())-len(selectedTargets) > 0:
print("%d target(s) excluded (unless required as dependencies): "%(len(init.targets())-len(selectedTargets)), file=stdout)
for t in sorted([' %-15s %s'%('<'+t.type+'>', t.name) for t in init.targets().values() if t not in selectedTargets]):
print(t, file=stdout)
print(file=stdout)
print("%d target(s) included: "%(len(selectedTargets)), file=stdout)
for t in sorted([' %-15s %s'%('<'+t.type+'>', t.name) for t in selectedTargets]):
print(t, file=stdout)
print(file=stdout)
if allTargets:
print("%d tag(s) are defined: "%(len(init.tags())), file=stdout)
for t in sorted([' %-15s (%d targets)'%(t, len(init.tags()[t])) for t in init.tags()]):
print(t, file=stdout)
elif task == _TASK_LIST_TARGET_INFO:
if findTargetsList == '*': findTargetsList = init.targets().values()
for t in sorted(findTargetsList, key=lambda t:(t.type+' '+t.name)):
print('- %s priority: %s, tags: %s, location: \n %s'%(t, t.getPriority(), t.getTags(), t.location), file=stdout)
elif task == _TASK_LIST_FIND_TARGETS:
# sort matches at start of path first, then anywhere in name, finally anywhere in type
for t in findTargetsList:
# this must be very easy to copy+paste, so don't put anything else on the line at all
print('%s'%(t.name), file=stdout)
elif task == _TASK_LIST_SEARCH:
def showPatternMatches(x): # case sensitive is fine (and probably useful)
if searchPattern.replace('\\', '/') in x.replace('\\','/'): return True # basic substring check (with path normalization)
if '*' in searchPattern or '?' in searchPattern or '[' in searchPattern: # probably a regex
if re.search(searchPattern, x): return True
return False
for t in init.targets().values():
t._resolveTargetPath(init)
print('', file=stdout)
tagMatches = [t for t in init.tags() if showPatternMatches(t)]
if tagMatches:
print ('%d matching tags:'%len(tagMatches), file=stdout)
for t in sorted(tagMatches):
print(t, file=stdout)
print('', file=stdout)
targetMatches = [t for t in init.targets().values() if showPatternMatches(t.name) or showPatternMatches(t.path)]
if targetMatches:
print ('%d matching targets:'%len(targetMatches), file=stdout)
for t in sorted(targetMatches, key=lambda t:(t.type+' '+t.name)):
print('- %s priority: %s, tags: [%s]\n output: %s\n defined: %s'%(t, t.getPriority(), ' '.join(sorted(t.getTags())) or 'none', os.path.relpath(t.path), t.location), file=stdout)
print('', file=stdout)
propMatches = {key:value for (key,value) in init.getProperties().items() if showPatternMatches(key)}
if propMatches:
print('%d matching properties:'%len(propMatches), file=stdout)
pad = max(list(map(len, propMatches.keys())))
for k in sorted(propMatches.keys()):
print(('%'+str(pad)+'s = %s') % (k, propMatches[k]), file=stdout)
if init._propertyLocations[k]: # don't do this for built-in property like BUILD_MODE
print(('%'+str(pad)+'s (defined: %s)') % ('', init._propertyLocations[k]), file=stdout)
options = init.mergeOptions(None)
optionMatches = {key:value for (key,value) in options.items() if showPatternMatches(key)}
if optionMatches:
print('%d matching options:'%len(optionMatches), file=stdout)
pad = max(list(map(len, optionMatches.keys())))
for k in sorted(optionMatches.keys()):
print(('%'+str(pad)+'s = %s') % (k, optionMatches[k]), file=stdout)
elif task in [_TASK_BUILD, _TASK_CLEAN, _TASK_REBUILD]:
if not logFile:
if allTargets:
buildtag = None
else:
buildtag = 'custom'
logFile = _maybeCustomizeLogFilename(init.getPropertyValue('LOG_FILE'),
buildtag,
task==_TASK_CLEAN)
logFile = os.path.abspath(logFile)
logdir = os.path.dirname(logFile)
if logdir and not os.path.exists(logdir): mkdir(logdir)
log.critical('Writing build log to: %s', os.path.abspath(logFile))
# also buffer the .log file, since it's just a lot harder to read when multiple target lines are all jumbled up;
# we have an undocumented env var for disabling this in case of debugging
if os.getenv('XPYBUILD_LOGFILE_OUTPUT_BUFFERING_DISABLED','')=='true': outputBufferingDisabled = True
logFileStream = OutputBufferingStreamWrapper(open(logFile, 'w', encoding='UTF-8'), bufferingDisabled=outputBufferingDisabled)
hdlr = logging.StreamHandler(logFileStream)
hdlr.setFormatter(logging.Formatter('%(asctime)s %(relativeCreated)05d %(levelname)-8s [%(threadName)s %(thread)5d] %(name)-10s - %(message)s', None))
hdlr.setLevel(logLevel or logging.INFO)
logging.getLogger().addHandler(hdlr)
log.info('Using xpybuild %s from %s on Python %s.%s.%s', XPYBUILD_VERSION, os.path.normpath(os.path.dirname(__file__)), sys.version_info[0], sys.version_info[1], sys.version_info[2])
log.info('Using build options: %s (logfile target outputBuffering=%s, stdout target outputBuffering=%s)', buildOptions, not outputBufferingDisabled, not wrapper.bufferingDisabled)
try:
# sometimes useful to have this info available
import socket, getpass
log.info('Build running on %s as user %s', socket.gethostname(), getpass.getuser())
except Exception as e:
log.info('Failed to get host/user: %s', e)
log.info('Default encoding for subprocesses assumed to be: %s (stdout=%s, preferred=%s)',
DEFAULT_PROCESS_ENCODING, stdout.encoding, locale.getpreferredencoding())
def lowerCurrentProcessPriority():
if xpybuild.buildcommon.IS_WINDOWS:
import win32process, win32api,win32con
win32process.SetPriorityClass(win32api.GetCurrentProcess(), win32process.BELOW_NORMAL_PRIORITY_CLASS)
else:
# on unix, people may run nice before executing the process, so
# only change the priority unilaterally if it's currently at its
# default value
if os.nice(0) == 0:
os.nice(1) # change to 1 below the current level
try:
# if possible, set priority of builds to below normal by default,
# to avoid starving machines (e.g. on windows) of resources
# that should be used for interactive processes
if os.getenv('XPYBUILD_DISABLE_PRIORITY_CHANGE','') != 'true':
lowerCurrentProcessPriority()
log.info('Successfully changed process priority to below normal')
except Exception as e:
log.warning('Failed to lower current process priority: %s'%e)
if buildOptions['ignore-deps']:
log.warning('The ignore-deps option is enabled: dependency graph will be ignored for all targets that already exist on disk, so correctness is not guaranteed')
for (k,v) in sorted(init.getProperties().items()):
log.info('Setting property %s=%s', k, v)
try:
DATE_TIME_FORMAT = "%a %Y-%m-%d %H:%M:%S %Z"
errorsList = []
if task in [_TASK_CLEAN, _TASK_REBUILD]:
startTime = time.time()
log.critical('Starting "%s" clean "%s" at %s', init.getPropertyValue('BUILD_MODE'), init.getPropertyValue('BUILD_NUMBER'),
time.strftime(DATE_TIME_FORMAT, time.localtime( startTime )))
cleanBuildOptions = buildOptions.copy()
cleanBuildOptions['clean'] = True
if allTargets: cleanBuildOptions['ignore-deps'] = True
scheduler = BuildScheduler(init, selectedTargets, cleanBuildOptions)
errorsList, targetsBuilt, targetsCompleted, totalTargets = scheduler.run()
if allTargets and not cleanBuildOptions['dry-run']: # special-case this common case
for dir in init.getOutputDirs():
deleteDir(dir)
log.critical('Completed "%s" clean "%s" at %s after %s\n', init.getPropertyValue('BUILD_MODE'), init.getPropertyValue('BUILD_NUMBER'),
time.strftime(DATE_TIME_FORMAT, time.localtime( startTime )), formatTimePeriod(time.time()-startTime))
if errorsList:
log.critical('XPYBUILD FAILED: %d error(s): \n %s', len(errorsList), '\n '.join(sorted(errorsList)))
return 3
if task == _TASK_REBUILD:
# we must reload the build file here, as it's the only way of flushing out
# cached data (especially in PathSets) that may have changed as a
# result of the clean
init = loadBuildFile()
if task in [_TASK_BUILD, _TASK_REBUILD] and not errorsList:
for cb in init.getPreBuildCallbacks():
try:
cb(BuildContext(init))
except BuildException as be:
log.error("Pre-build check failed: %s", be)
return 7
buildtype = 'incremental' if any(os.path.exists(dir) for dir in init.getOutputDirs()) else 'fresh'
if not buildOptions['dry-run']:
for dir in init.getOutputDirs():
log.info('Creating output directory: %s', dir)
mkdir(dir)
startTime = time.time()
log.critical('Starting %s "%s" build "%s" at %s using %d workers', buildtype,
init.getPropertyValue('BUILD_MODE'), init.getPropertyValue('BUILD_NUMBER'),
time.strftime(DATE_TIME_FORMAT, time.localtime( startTime )),
buildOptions['workers']
)
buildOptions['clean'] = False
scheduler = BuildScheduler(init, selectedTargets, buildOptions)
errorsList, targetsBuilt, targetsCompleted, totalTargets = scheduler.run()
log.critical('Completed %s "%s" build "%s" at %s after %s\n', buildtype, init.getPropertyValue('BUILD_MODE'), init.getPropertyValue('BUILD_NUMBER'),
time.strftime(DATE_TIME_FORMAT, time.localtime( startTime )), formatTimePeriod(time.time()-startTime))
if 'timeFile' in buildOptions:
logTargetTimes(buildOptions['timeFile'], scheduler, init)
if errorsList:
# heuristically: it's useful to have them in order of failure when a small number, but if there are
# lots then it's too hard to read and better to sort, so similar ones are together
if len(errorsList)>=10: errorsList.sort()
log.critical('*** XPYBUILD FAILED: %d error(s) (aborted with %d targets outstanding): \n %s', len(errorsList), totalTargets-targetsCompleted, '\n '.join(errorsList))
return 4
else:
# using *** here means we get a valid final progress message
log.critical('*** XPYBUILD SUCCEEDED: %s built (%d up-to-date)', targetsBuilt if targetsBuilt else '<NO TARGETS>', (totalTargets-targetsBuilt))
return 0
finally:
publishArtifact('Xpybuild logfile', logFile)
else:
raise Exception('Task type not implemented yet - '+task) # should not happen
except BuildException as e:
# hopefully we don't end up here very often
log.error('*** XPYBUILD FAILED: %s', e.toMultiLineString(None))
return 5
except Exception as e:
log.exception('*** XPYBUILD FAILED: ')
return 6
def clean(self, context):
if self.cleanfn: self.cleanfn(self.path, context)
BaseTarget.clean(self, context)