def run(self, context):
mkdir(os.path.dirname(self.path))
src = self.src.resolve(context)
if len(src) != 1: raise BuildException('SymLink target "%s" is invalid - must have only only source path'%self.name)
if src[0].endswith('/'): raise BuildException('SymLink target "%s" is invalid - must be a file not a directory'%self.name)
os.symlink(src[0] if not self.relative else os.path.relpath(src[0], os.path.dirname(self.path)), self.path)
def run(self, context):
self.log.info('Cleaning existing files from %s', self.path)
deleteDir(self.path)
iswindows = IS_WINDOWS
for a in self.archives:
a_is_filteredarchivecontents = isinstance(a, FilteredArchiveContents)
if a_is_filteredarchivecontents:
items = [(a.getResolvedPath(context), '')]
else:
assert isinstance(a, BasePathSet)
filteredMembers = None
items = a.resolveWithDestinations(context)
for (srcAbs, destRel) in items:
if destRel and not isDirPath(destRel): destRel = os.path.dirname(destRel) # strip off the zip filename
if '..' in destRel: raise Exception('This target does not permit destination paths to contain ".." relative path expressions')
try:
filesize = os.path.getsize(srcAbs)
except Exception:
filesize = 0
self.log.info("Unpacking %s (%0.1f MB) to %s", os.path.basename(srcAbs), filesize/1024.0/1024, self.name+destRel)
starttime = time.time()
with self. __openArchive(srcAbs) as f:
mkdir(self.path+destRel)
if a_is_filteredarchivecontents and a.hasIncludeExcludeFilters():
fullList = _getnames(f)
if not fullList:
raise BuildException('No files were found in archive "%s"'%(srcAbs))
filteredMembers = [x for x in fullList if a.isIncluded(context, x)]
self.log.info("Unpacking %d of %d members in %s", len(filteredMembers), len(fullList), os.path.basename(srcAbs))
if not filteredMembers:
raise BuildException('No files matching the specified include/exclude filters were found in archive "%s": %s'%(srcAbs, a))
if len(filteredMembers)==len(fullList):
raise BuildException('No files were excluded from the unpacking operation by the specified filters (check filters are correct): %s'%a)
else:
filteredMembers = _getnames(f)
# NB: some archive types want a list of string members, others want TarInfo objects etc, so
# if we support other archive types in future might need to do a bit of work here
path = normLongPath(self.path+destRel)
for m in filteredMembers:
if not isDirPath(m):
info = _getinfo(f, m)
if a_is_filteredarchivecontents:
_setfilename(info, a.mapDestPath(context, _getfilename(info)))
if iswindows: _setfilename(info, _getfilename(info).replace('/', '\\'))
f.extract(info, path=path)
else:
# we should create empty directories too
if a_is_filteredarchivecontents:
m = a.mapDestPath(context, m).rstrip('/')
m = path.rstrip('/\\')+'/'+m
if iswindows: m = m.replace('/', '\\')
mkdir(m)
self.log.info("Completed unpacking %s (%0.1f MB) in %0.1f seconds", os.path.basename(srcAbs), filesize/1024.0/1024, (time.time()-starttime))
def run(self, context):
libs = self.libs.resolve(context)
libnames = [os.path.basename(x) for x in libs]
libpaths = [os.path.dirname(x) for x in libs]
flags = [context.expandPropertyValues(x) for x in self.flags]
args = [self.getOption('csharp.compiler'), "-out:" + self.path]
if libnames:
args.extend([
"-reference:" + ",".join(libnames),
"-lib:" + ",".join(libpaths)
])
if self.main:
args.extend(["-target:exe", "-main:" + self.main])
else:
args.append("-target:library")
for (file, id) in self.resources:
args.append('-resource:%s,%s' %
(context.expandPropertyValues(file),
context.expandPropertyValues(id)))
args.extend(self.options['csharp.options'])
args.extend(flags)
args.extend(self.compile.resolve(context))
mkdir(os.path.dirname(self.path))
call(args,
outputHandler=self.getOption('csharp.outputHandlerFactory')(
'csc', False, options=self.options),
timeout=self.options['process.timeout'])
def run(self, context):
"""
Calls the wrapped run method
"""
retries = None
retryNumber = 0 # 1=first retry, etc
self.target.retriesRemaining = int(self.target.options['Target.failureRetries']) # default is 0
backoffSecs = self.target.options['Target.failureRetriesInitialBackoffSecs']
while True:
try:
implicitInputs = self.__getImplicitInputs(context)
if implicitInputs or self.isDirPath:
deleteFile(self.__implicitInputsFile)
self.target.run(context)
if retryNumber > 0: self.target.log.warning('Target %s succeeded on retry #%d', self, retryNumber)
break
except Exception as ex:
if self.target.retriesRemaining == 0:
if retryNumber > 0:
self.target.log.warning('Target %s failed even after %d retries', self, retryNumber)
raise
self.target.retriesRemaining -= 1
retryNumber += 1
time.sleep(backoffSecs)
self.target.log.info('Target %s failed on attempt #%d, will now clean before retry', self, retryNumber)
# hopefully after the backoff time enough file handles will have been removed for the clean to succeed
try:
self.clean(context)
except Exception as ex:
self.target.log.error('Failed to cleanup during retry, after initial failure of %s', self)
raise
# this logic is to prevent CI (e.g. TeamCity) error messages from one retry from causing the whole job to be flagged as a
# failure even if a subsequent retry succeeds
buf = outputBufferingManager.resetStdoutBufferForCurrentThread()
self.target.log.warning('Target %s failed on attempt #%d, will retry after %d seconds backoff (see .log file for details).', self, retryNumber, backoffSecs)
# for now let's just throw away buf - user can look at the .log file to see what happened before the failure if they care
# (can't just re-log it here as that would result in duplication with the .log output)
backoffSecs *= 2
# we expect self.path to NOT be in the fileutils stat cache at this point;
# it's too costly to check this explictly, but unwanted incremental rebuilds
# can be caused if the path does get into the stat cache since it'll have a stale value
# if target built successfully, record what the implicit inputs were to help with the next up to date
# check and ensure incremental build is correct
if implicitInputs or self.isDirPath:
log.debug('writing implicitInputsFile: %s', self.__implicitInputsFile)
mkdir(os.path.dirname(self.__implicitInputsFile))
with openForWrite(self.__implicitInputsFile, 'wb') as f:
f.write(os.linesep.join(implicitInputs).encode('utf-8'))
def run(self, context):
options = self.options
mkdir(os.path.dirname(self.path))
options['native.compilers'].archiver.archive(
context,
output=self.path,
options=options,
src=self.objects.resolve(context))
def run(self, context):
mkdir(self.path)
if os.listdir(self.path):
self.log.error('TEST FAILED - clean did not occur between retries')
open(self.path + '/touchfile.txt', 'w').close()
self.log.error('Error logged by target')
raise Exception('Simulated target failure')
def updateStampFile(self):
"""
.. private:: Not useful enough to be in the public API.
Assumes self.path is a stamp file that just needs creating / timestamp updating and does so """
path = normLongPath(self.path)
mkdir(os.path.dirname(path))
with openForWrite(path, 'wb') as f:
pass
def run(self, context):
options = self.options
mkdir(os.path.dirname(self.path))
options['native.compilers'].ccompiler.compile(
context,
output=self.path,
options=options,
flags=self._getCompilerFlags(context),
src=self.source.resolve(context),
includes=self._getIncludeDirs(context))
def run(self, context):
# make sure outputdir exists
mkdir(self.path)
# create the classpath, sorting within PathSet (for determinism), but retaining original order of
# PathSet elements in the list
classpath = os.pathsep.join(self.classpath.resolve(context))
# compile everything
mkdir(self.getOption('javac.logs'))
javac(self.path, self.compile.resolve(context), classpath, options=self.options, logbasename=self.options['javac.logs']+'/'+targetNameToUniqueId(self.name), targetname=self.name, workDir=self.workDir)
def run(self, context):
mkdir(os.path.dirname(self.path))
alreadyDone = set()
with zipfile.ZipFile(normLongPath(self.path), 'w') as output:
for (f, o) in self.inputs.resolveWithDestinations(context):
# if we don't check for duplicate entries we'll end up creating an invalid zip
if o in alreadyDone:
dupsrc = ['"%s"'%src for (src, dest) in self.inputs.resolveWithDestinations(context) if dest == o]
raise BuildException('Duplicate zip entry "%s" from: %s'%(o, ', '.join(dupsrc)))
alreadyDone.add(o)
# can't compress directory entries! (it messes up Java)
output.write(normLongPath(f).rstrip('/\\'), o, zipfile.ZIP_STORED if isDirPath(f) else zipfile.ZIP_DEFLATED)
def jar(path, manifest, sourcedir, options, preserveManifestFormatting=False, update=False, outputHandler=None): """ Create a jar file containing a manifest and some other files @param path: jar file to create. Typically this file does not already exist, but if it does then the specified files or manifest will be merged into it. @param manifest: path to the manifest.mf file (or None to disable manifest entirely) @param sourcedir: the directory to pack everything from (this method may add extra files to this dir) @param options: options map. jar.options is a list of additional arguments @param preserveManifestFormatting: an advanced option that prevents that jar executable from reformatting the specified manifest file to comply with Java conventions (also prevents manifest merging if jar already exists) """ # work out if we need to create a parent directory dir = os.path.dirname(path) if dir and not os.path.exists(dir): mkdir(dir) # location of jar if options['java.home']: binary = os.path.join(options['java.home'], "bin/jar") else: binary = "jar" # build up arguments args = [binary] args.extend(options['jar.options']) if update: mode='-u' else: mode='-c' if not manifest: args.extend([mode+"fM", path]) elif preserveManifestFormatting: mkdir(sourcedir+'/META-INF') srcf = normLongPath(sourcedir+'/META-INF/manifest.mf') with open(manifest, 'rb') as s: with openForWrite(srcf, 'wb') as d: d.write(s.read()) args.extend([mode+"f", path]) else: args.extend([mode+"fm", path, manifest]) if sourcedir: args.extend(["-C", sourcedir, "."]) # actually call jar call(args, outputHandler=outputHandler, timeout=options['process.timeout'])
def run(self, context):
contents = self._getContents(context)
mkdir(os.path.dirname(self.path))
path = normLongPath(self.path)
with self.openFile(context,
path,
'wb' if isinstance(contents, bytes) else 'w',
encoding=self.__encoding) as f:
f.write(contents)
if self.__mode and not IS_WINDOWS:
os.chmod(path, self.__mode)
if self.__executable and not IS_WINDOWS:
os.chmod(
path, stat.S_IXOTH | stat.S_IXUSR | stat.S_IXGRP
| os.stat(self.path).st_mode)
def javadoc(path, sources, classpath, options, workDir, outputHandler):
""" Create javadoc from sources and a set of options
@param path: The directory under which to create the javadoc
@param sources: a list of source files
@param classpath: a list of jars for the classpath
@param options: the current set of options to use
@param workDir: where temporary files are stored
@param outputHandler: the output handler (optional)
"""
deleteDir(path)
mkdir(path)
# location of javadoc
if options['java.home']:
binary = os.path.join(options['java.home'], "bin/javadoc")
else:
binary = "javadoc"
# store the list of files in a temporary file, then build from that.
mkdir(workDir)
inputlistfile = os.path.join(workDir, "javadoc.inputs")
with openForWrite(inputlistfile, 'w', encoding=locale.getpreferredencoding()) as f:
f.writelines('"'+x.replace('\\','\\\\')+'"'+'\n' for x in sources)
# build up arguments
args = [binary]
args.extend(options['javadoc.options'])
if options['javadoc.ignoreSourceFilesFromClasspath']:
args.extend(['-sourcepath', path+'/xpybuild_fake_sourcepath'])
args.extend([
"-d", path,
"-classpath", classpath,
"-windowtitle", options['javadoc.title'],
"-doctitle", options['javadoc.title'],
"-%s" % options['javadoc.access'],
"@%s" % inputlistfile
])
# actually call javadoc
call(args, outputHandler=outputHandler, timeout=options['process.timeout'])
def run(self, context):
options = self.options
mkdir(os.path.dirname(self.path))
options['native.compilers'].linker.link(
context,
output=self.path,
options=options,
flags=options['native.link.flags'] + self.flags,
shared=self.shared,
src=self.objects.resolve(context),
libs=flatten([
(y.strip()
for y in context.expandPropertyValues(x, expandList=True))
for x in self.libs + options['native.libs'] if x
]),
libdirs=flatten(
self.libpaths.resolve(context) + [
context.expandPropertyValues(x, expandList=True)
for x in options['native.libpaths']
]))
def run(self, context):
mkdir(os.path.dirname(self.path))
with tarfile.open(normLongPath(self.path), 'w:gz') as output:
for (f, o) in self.inputs.resolveWithDestinations(context):
output.add(normLongPath(f).rstrip('/\\'), o)
def getHashableImplicitInputs(self, context):
r = super(Cpp, self).getHashableImplicitInputs(context)
r.append('compiler flags: %s' % self._getCompilerFlags(context))
# this will provide a quick way to notice changes such as TP library version number changed etc
# and also is the only way we'll detect the need to rebuild for includes that are regex'd out
includedirs = self._getIncludeDirs(context)
for path in includedirs:
r.append('include dir: ' + os.path.normcase(path))
# This is called exactly once during up-to-date checking OR run, which
# means we will have generated all target dependencies
# (e.g. include files, source files etc) by this point
# Since non-target include files won't be known until this point, we need
# to perform up-to-date-ness checking for them here (rather than in
# targetwrapper as normally happens for dependencies).
startt = time.time()
try:
targetmtime = os.stat(
self.path
).st_mtime # must NOT use getstat cache, don't want to pollute it with non-existence
except os.error: # file doesn't exist
targetmtime = 0
makedependsfile = self._getMakeDependsFile(context)
if targetmtime != 0 and not os.path.exists(
makedependsfile
): # no value in using stat cache for this, not used elsewhere
targetmtime = 0 # treat the same as if target itself didn't exist
newestFile, newestTime = None, 0 # keep track of the newest source or include file
if IS_WINDOWS: # normalizes case for this OS but not slashes (handy for regex matching)
def xpybuild_normcase(path):
return path.lower()
else:
def xpybuild_normcase(path):
return path
# changes in these options must cause us to re-execute makedepends
ignoreregex = self.options['native.include.upToDateCheckIgnoreRegex']
if ignoreregex:
ignoreregex = xpybuild_normcase(ignoreregex)
r.append('option native.include.upToDateCheckIgnoreRegex=%s' %
ignoreregex)
makedependsoptions = "upToDateCheckIgnoreRegex='%s', upToDateCheckIgnoreSystemHeaders=%s, flags=%s" % (
ignoreregex,
self.options['native.include.upToDateCheckIgnoreSystemHeaders'],
self._getCompilerFlags(context),
)
# first, figure out if we need to (re-)run makedepends or can use the cached info from the last build
runmakedepends = False
if targetmtime == 0:
runmakedepends = True
alreadychecked = set() # paths that we've already checked the date of
sourcepaths = []
for path, _ in self.source.resolveWithDestinations(context):
mtime = cached_getmtime(path)
alreadychecked.add(path)
sourcepaths.append(path)
if mtime > newestTime: newestFile, newestTime = path, mtime
if newestTime > targetmtime: runmakedepends = True
if (not runmakedepends) and os.path.exists(
makedependsfile): # (no point using stat cache for this file)
# read file from last time; if any of the transitive dependencies
# have changed, we should run makedepends again to update them
with io.open(makedependsfile, 'r', encoding='utf-8') as f:
flags = f.readline().strip()
if flags != makedependsoptions:
runmakedepends = True
else:
for path in f:
path = path.strip()
pathstat = cached_stat(path, errorIfMissing=False)
if pathstat is False:
# file doesn't exist - must rebuild
runmakedepends = True
(
self.log.critical
if Cpp.__rebuild_makedepend_count <= 5 else
self.log.info
)('Recalculating C/C++ dependencies of %s as dependency no longer exists: %s',
self, newestFile)
break
mtime = pathstat.st_mtime
alreadychecked.add(path)
if mtime > newestTime:
newestFile, newestTime = path, mtime
if newestTime > targetmtime: runmakedepends = True
# (re-)run makedepends
if runmakedepends:
# only bother to log if we're recalculating
if targetmtime != 0:
Cpp.__rebuild_makedepend_count += 1 # log the first few at crit
(
self.log.critical
if Cpp.__rebuild_makedepend_count <= 5 else self.log.info
)('Recalculating C/C++ dependencies of %s; most recently modified dependent file is %s at %s',
self, newestFile, datetime.datetime.fromtimestamp(
newestTime).strftime('%a %Y-%m-%d %H:%M:%S'))
try:
makedependsoutput = self.options[
'native.compilers'].dependencies.depends(
context=context,
src=sourcepaths,
options=self.options,
flags=self._getCompilerFlags(context),
includes=includedirs,
)
except Exception as ex:
raise BuildException('Dependency resolution failed for %s' %
(sourcepaths[0]),
causedBy=True)
# normalize case to avoid problems on windows, and strip out sources since we already checked them above
makedependsoutput = [
os.path.normcase(path) for path in makedependsoutput
if path not in sourcepaths
]
makedependsoutput.sort()
if ignoreregex:
ignoreregex = re.compile(ignoreregex)
# match against version of path with forward slashes because making a regex with backslashes is a pain and not cross-platform
makedependsoutput = [
path for path in makedependsoutput
if not ignoreregex.match(path.replace(os.sep, '/'))
]
# find the newest time from these files; if this is same as previous makedepends, won't do anything
for path in makedependsoutput:
if path in alreadychecked: continue
mtime = cached_getmtime(path)
if mtime > newestTime: newestFile, newestTime = path, mtime
# write out new makedepends file for next time
mkdir(os.path.dirname(makedependsfile))
assert '\n' not in makedependsoptions, makedependsoptions # sanity check
with io.open(makedependsfile, 'w', encoding='utf-8') as f:
f.write(makedependsoptions)
f.write('\n')
for path in makedependsoutput:
f.write('%s\n' % path)
# endif runmakedepends
# include the newest timestamp as an implicit input, so that we'll rebuild if any include files have changed
# no need to log this, as targetwrapper already logs differences in implicit inputs
if newestFile is not None:
newestDateTime = datetime.datetime.fromtimestamp(newestTime)
r.append('newest dependency was modified at %s.%03d: %s' %
(newestDateTime.strftime('%a %Y-%m-%d %H:%M:%S'),
newestDateTime.microsecond / 1000,
os.path.normcase(newestFile)))
if time.time(
) - startt > 5: # this should usually be pretty quick, so if it takes a while it may indicate a real build file mistake
self.log.warn(
'C/C++ dependency generation took a long time: %0.1f s to evaluate %s',
time.time() - startt, self)
return r
def _expand_deps(self):
"""
Run over the list of targets to build, expanding it with all the dependencies and processing them
for replacements and expansions. Also builds up the initial leaf set self.leaves and all the rdepends of each
target (self.pending), along with the total number of dependencies each target has.
"""
self.index = 0 # identifies thread pool item n out of total=len(self.pending)
self.total = len(self.pending) # can increase during this phase
pending = queue.Queue()
for i in self.pending:
pending.put_nowait((0, i))
depcheckingworkers = 1 # more threads actually makes this slower
pool = ThreadPool('dependencychecking',
depcheckingworkers,
pending,
self._deps_target,
self.utilisation,
profile=self.options["profile"])
pool.start()
pool.wait()
pool.stop()
# these are GIL-intensive operation
if (not self.options['clean']) and len(pool.errors) == 0:
for targetwrapper, _ in self.selectedtargetwrappers:
targetwrapper.updatePriority()
targetwrapper.checkForNonTargetDependenciesUnderOutputDirs()
if not pool.errors:
# this is the only place we can list all targets since only here are final priorities known
# printing the deps in one place is important for debugging missing dependencies etc
# might move this to a separate file at some point
self.selectedtargetwrappers.sort(
key=lambda targetwrapper_targetdeps:
(-targetwrapper_targetdeps[0].effectivePriority,
targetwrapper_targetdeps[0].name))
targetinfodir = mkdir(
self.context.expandPropertyValues(
'${BUILD_WORK_DIR}/targets/'))
with io.open(targetinfodir + '/xpybuild-version.properties',
'w',
encoding='utf-8') as f:
# write this file in case we want to avoid mixed xpybuild versions in working dir
f.write('xpybuildVersion=%s\n' % XPYBUILD_VERSION)
f.write(
'workDirVersion=%d\n' % 1
) # bump this when we make a breaking change that should force a rebuild
with io.open(targetinfodir + '/selected-targets.txt',
'w',
encoding='utf-8') as f:
f.write('%d targets selected for building:\n' %
(len(self.selectedtargetwrappers)))
for targetwrapper, targetdeps in self.selectedtargetwrappers:
f.write('- Target %s with priority %s depends on: %s\n\n' %
(targetwrapper, targetwrapper.effectivePriority,
', '.join(str(d) for d in targetdeps)
if len(targetdeps) > 0 else '<no dependencies>'))
#assert (not pool.errors) or (self.total == self.index), (self.total, self.index) #disabled because assertion triggers during ctrl+c
return pool.errors
def main(args):
""" Command line argument parser.
"""
try:
usage = [
###############################################################################
'',
'eXtensible Python-based Build System %s on Python %s.%s.%s'% (XPYBUILD_VERSION, sys.version_info[0], sys.version_info[1], sys.version_info[2]),
'',
'xpybuild.py [operation]? [options]* [property=value]* [-x] [target|tag|regex]* ',
'',
'A regex containing * can be used instead of a target, but only if it uniquely ',
'identifies a single target. ',
'',
'Special pseudo-tags:',
' full Include all targets for a full build (the default)',
'',
'Special properties:',
' OUTPUT_DIR=output The main directory output will be written to',
' BUILD_MODE=release Specifies release, debug (or user-defined) mode',
' BUILD_NUMBER=n Build number string, for reporting and use by build',
'',
'Operations: ',
###############################################################################
' (if none is specified, the default operation is a normal build)',
' --clean Clean specified targets incl all deps',
' --rebuild Clean specified targets incl all deps then build',
' --rebuild-ignore-deps Clean only the specified targets (not deps) then ',
' (or --rid) build those targets and any missing dependencies, ',
' but not any out-of-date dependencies. This is a ',
' fast but less correct way to get a quick ',
' incremental build, so use with care. ',
'',
' -s --search <str> Show info on targets/tags/properties/options ',
' containing the specified substring or regex',
# hide these from usage (though they still work), as superceded by the more useful "-s" option
#' --ft --find-targets <str> List targets containing the specified substring',
#' --ti --target-info <str> Print details including build file location for ',
#' targets containing the specified substring',
' --targets List available targets and tags (filtered by any ',
' target or tag names specified on the command line)',
' --properties List properties that can be set and their ',
' defaults in this build file',
' --options List the target options available to build rules ',
' and their default values in this build file',
'',
'Options:',
' -x --exclude <target> Specifies a target or tag to exclude (unless ',
' needed as a dependency of an included target) ',
'',
' -J --parallel Build in parallel (this is the default). ',
' The number of workers is determined from the ',
' `build.workers` build file option or else the ',
' number of CPUs and the XPYBUILD_WORKERS_PER_CPU ',
' environment varable (default is currently 1.0), ',
' with an upper limit for this machine from the ',
' XPYBUILD_MAX_WORKERS variable. ',
' -j --workers <number> Override the number of worker threads to use for ',
' building. Use -j1 for single-threaded. ',
' (ignores any environment variables)',
'',
' -k --keep-going Continue rather than aborting on errors',
'',
' -n --dry-run Don\'t actually build anything, just print',
' what would be done (finds missing dependencies)',
'',
' --id --ignore-deps Skip all dependency/up-to-date checking: only ',
' clean/build targets that do not exist at all ',
' (faster builds, but no guarantee of correctness)',
'',
' -f --buildfile <file> Specify the root build file to import ',
' (default is ./root.xpybuild.py)',
'',
' -l --log-level LEVEL Set the log level to debug/info/critical',
' -L --logfile <file> Set the log file location',
' --timefile <file> Dump the time for each target in <file> at the',
' end of the run',
' --depgraph <file> Just resolve dependencies and dump them to <file>',
' --cpu-stats Log CPU utilisation stats',
' --random-priority Randomizes build order',
' --verify Performs additional verifications during the ',
' build to to help detect bugs in the build files. ',
' [verify is currently an experimental feature]',
' --profile Profiles all the worker threads',
' -F --format Message output format.',
' Options:',
] + [
' - '+ h for h in _registeredConsoleFormatters
] + [
]
if reduce(max, list(map(len, usage))) > 80:
raise Exception('Invalid usage string - all lines must be less than 80 characters')
# set up defaults
properties = {}
buildOptions = { "keep-going":False, "workers":0, "dry-run":False,
"ignore-deps":False, "logCPUUtilisation":False, "profile":False, "verify":False }
includedTargets = []
excludedTargets = []
task = _TASK_BUILD
buildFile = os.path.abspath('root.xpybuild.py')
logLevel = None
logFile = None
findTargetsPattern = None
format = "default"
opts,targets = getopt.gnu_getopt(args, "knJh?x:j:l:L:f:F:s:",
["help","exclude=","parallel","workers=","keep-going",
"log-level=","logfile=","buildfile=", "dry-run",
"targets", 'target-info=', 'ti=', "properties", "options", "clean", "rebuild", "rebuild-ignore-deps", "rid", "ignore-deps", "id",
"format=", "timefile=", "ft=", "find-targets=", "search=", "depgraph=", 'cpu-stats', 'random-priority', 'profile', 'verify'])
for o, a in opts: # option arguments
o = o.strip('-')
if o in ["?", "h", "help"]:
print('\n'.join(usage))
return 0
elif o in ["x", "exclude"]:
excludedTargets.append(a)
elif o in ["f", "buildfile"]:
buildFile = os.path.abspath(a)
elif o in ['targets']:
task = _TASK_LIST_TARGETS
elif o in ['find-targets', 'ft']:
task = _TASK_LIST_FIND_TARGETS
findTargetsPattern = BaseTarget._normalizeTargetName(a)
elif o in ['target-info', 'ti']:
task = _TASK_LIST_TARGET_INFO
findTargetsPattern = BaseTarget._normalizeTargetName(a)
elif o in ['search', 's']:
task = _TASK_LIST_SEARCH
searchPattern = BaseTarget._normalizeTargetName(a)
elif o in ['properties']:
task = _TASK_LIST_PROPERTIES
elif o in ['options']:
task = _TASK_LIST_OPTIONS
elif o in ['J', 'parallel']:
buildOptions['workers'] = 0
elif o in ['j', 'workers']:
buildOptions['workers'] = int(a)
elif o in ['l', 'log-level']:
logLevel = getattr(logging, a.upper(), None)
elif o in ['cpu-stats']:
buildOptions["logCPUUtilisation"] = True
elif o in ['random-priority']:
buildOptions["randomizePriorities"] = True
elif o in ['L', 'logfile']:
logFile = a
elif o in ['F', 'format']:
format = None
if a =='xpybuild': a = 'default' # for compatibility
for h in _registeredConsoleFormatters:
if h.upper() == a.upper():
format = h
if not format:
print('invalid format "%s"; valid formatters are: %s'%(a, ', '.join(_registeredConsoleFormatters.keys())))
print('\n'.join(usage))
return 1
elif o in ['clean']:
task = _TASK_CLEAN
buildOptions['keep-going'] = True
elif o in ['rebuild']:
task = _TASK_REBUILD
elif o in ['rebuild-ignore-deps', 'rid']:
task = _TASK_REBUILD
buildOptions['ignore-deps'] = True
elif o in ['id', 'ignore-deps']:
buildOptions['ignore-deps'] = True
elif o in ['k', 'keep-going']:
buildOptions['keep-going'] = True
elif o in ['n', 'dry-run']:
buildOptions['dry-run'] = True
elif o in ['timefile']:
buildOptions['timeFile'] = a
elif o in ['verify']:
buildOptions['verify'] = True
elif o in ['profile']:
buildOptions['profile'] = True
elif o in ['depgraph']:
buildOptions['depGraphFile'] = a
else:
assert False, "unhandled option: '%s'" % o
for o in targets: # non-option arguments (i.e. no -- prefix)
arg = o.strip()
if arg:
if '=' in arg:
properties[arg.split('=')[0].upper()] = arg.split('=')[1]
elif arg=='all': # pre-4.0 alias for all
includedTargets.append('full')
else:
includedTargets.append(BaseTarget._normalizeTargetName(arg))
# default is all
if (not includedTargets) or includedTargets==['']:
includedTargets = ['full']
except getopt.error as msg:
print(msg)
print("For help use --help")
return 2
threading.currentThread().setName('main')
logging.getLogger().setLevel(logLevel or logging.INFO)
if buildOptions["workers"] < 0: buildOptions["workers"] = 0 # means there's no override
outputBufferingDisabled = buildOptions['workers']==1 # nb: this also affects the .log handler below
# nb: it's possible workers=0 (auto) and will later be set to 1 but doesn't really matter much
# initialize logging to stdout - minimal output to avoid clutter, but indicate progress
hdlr = _registeredConsoleFormatters.get(format, None)
assert hdlr # shouldn't happen
wrapper = OutputBufferingStreamWrapper(sys.stdout, bufferingDisabled=outputBufferingDisabled)
# actually instantiate it
hdlr = hdlr(
wrapper,
buildOptions=buildOptions)
# Handler can override wrapper with a preference in either direction
if hdlr.bufferingDisabled: wrapper.bufferingDisabled = True
if hdlr.bufferingRequired: wrapper.bufferingDisabled = False
hdlr.setLevel(logLevel or logging.WARNING)
logging.getLogger().addHandler(hdlr)
log.info('Build options: %s'%{k:buildOptions[k] for k in buildOptions if k != 'workers'})
stdout = sys.stdout
# redirect to None, to prevent any target code from doing 'print' statements - should always use the logger
sys.stdout = None
listen_for_stack_signal() # make USR1 print a python stack trace
allTargets = ('full' in includedTargets) and not excludedTargets
try:
def loadBuildFile():
init = BuildInitializationContext(properties)
isRealBuild = (task in [_TASK_BUILD, _TASK_CLEAN, _TASK_REBUILD])
init._defineOption("process.timeout", 600)
init._defineOption("build.keepGoing", buildOptions["keep-going"])
# 0 means default behaviour
init._defineOption("build.workers", 0)
init.initializeFromBuildFile(buildFile, isRealBuild=isRealBuild)
# now handle setting real value of workers, starting with value from build file
workers = int(init._globalOptions.get("build.workers", 0))
# default value if not specified in build file
if workers <= 0:
workers = multiprocessing.cpu_count()
if os.getenv('XPYBUILD_WORKERS_PER_CPU'):
workers = min(workers, int(round(multiprocessing.cpu_count() * float(os.getenv('XPYBUILD_WORKERS_PER_CPU')))))
# machine/user-specific env var can cap it
if os.getenv('XPYBUILD_MAX_WORKERS'):
workers = min(workers, int(os.getenv('XPYBUILD_MAX_WORKERS')))
# finally an explicit command line --workers take precedence
if buildOptions['workers']: workers = buildOptions['workers']
if workers < 1: workers = 1
# finally write the final number of workers where it's available to both scheduler and targets
buildOptions['workers'] = workers
init._globalOptions['build.workers'] = workers
init._finalizeGlobalOptions()
return init
if buildOptions['profile']:
import cProfile, pstats
profiler = cProfile.Profile()
profiler.enable()
init = loadBuildFile()
# nb: don't import any modules that might define options (including outputhandler)
# until build file is loaded
# or we may not have a build context in place yet#
from xpybuild.internal.scheduler import BuildScheduler, logTargetTimes
if buildOptions['profile']:
profilepath = 'xpybuild-profile-%s.txt'%'parsing'
with open(profilepath, 'w') as f:
p = pstats.Stats(profiler, stream=f)
p.sort_stats('cumtime').print_stats(f)
p.dump_stats(profilepath.replace('.txt', '')) # also in binary format
log.critical('=== Wrote Python profiling output to: %s', profilepath)
def lookupTarget(s):
tfound = init.targets().get(s,None)
if not tfound and '*' in s:
matchregex = s.rstrip('$')+'$'
try:
matchregex = re.compile(matchregex, re.IGNORECASE)
except Exception as e:
raise BuildException('Invalid target regular expression "%s": %s'%(matchregex, e))
matches = [t for t in init.targets().values() if matchregex.match(t.name)]
if len(matches) > 1:
print('Found multiple targets matching pattern %s:'%(s), file=stdout)
print(file=stdout)
for m in matches:
print(m.name, file=stdout)
print(file=stdout)
raise BuildException('Target regex must uniquely identify a single target: %s (use tags to specify multiple related targets)'%s)
if matches: return matches[0]
if not tfound: raise BuildException('Unknown target name, target regex or tag name: %s'%s)
return tfound
# expand tags to targets here, and do include/exclude calculations
selectedTargets = set() # contains BaseTarget objects
for t in includedTargets:
tlist = init.tags().get(t,None)
if tlist:
selectedTargets.update(tlist)
else:
selectedTargets.add(lookupTarget(t))
for t in excludedTargets:
tlist = init.tags().get(t,None)
if tlist:
selectedTargets.difference_update(tlist)
else:
selectedTargets.discard(lookupTarget(t))
# convert findTargetsPattern to list
if findTargetsPattern:
findTargetsPattern = findTargetsPattern.lower()
# sort matches at start of path first, then anywhere in name, finally anywhere in type
# make 'all'/'full' into a special case that maps to all *selected* targets
# (could be different to 'all' tag if extra args were specified, but this is unlikely and kindof useful)
findTargetsList = [t for t in sorted(
init.targets().values() if allTargets else selectedTargets, key=lambda t:(
'/'+findTargetsPattern.lower() not in t.name.lower(),
findTargetsPattern.lower() not in t.name.lower(),
findTargetsPattern.lower() not in t.type.lower(),
t.name
)) if findTargetsPattern in t.name.lower() or findTargetsPattern in t.type.lower()
or findTargetsPattern == 'full' or findTargetsPattern == 'all']
if task == _TASK_LIST_PROPERTIES:
p = init.getProperties()
print("Properties: ", file=stdout)
pad = max(list(map(len, p.keys())))
if pad > 30: pad = 0
for k in sorted(p.keys()):
print(('%'+str(pad)+'s = %s') % (k, p[k]), file=stdout)
elif task == _TASK_LIST_OPTIONS:
options = init.mergeOptions(None)
pad = max(list(map(len, options.keys())))
if pad > 30: pad = 0
for k in sorted(options.keys()):
print(("%"+str(pad)+"s = %s") % (k, options[k]), file=stdout)
elif task == _TASK_LIST_TARGETS:
if len(init.targets())-len(selectedTargets) > 0:
print("%d target(s) excluded (unless required as dependencies): "%(len(init.targets())-len(selectedTargets)), file=stdout)
for t in sorted([' %-15s %s'%('<'+t.type+'>', t.name) for t in init.targets().values() if t not in selectedTargets]):
print(t, file=stdout)
print(file=stdout)
print("%d target(s) included: "%(len(selectedTargets)), file=stdout)
for t in sorted([' %-15s %s'%('<'+t.type+'>', t.name) for t in selectedTargets]):
print(t, file=stdout)
print(file=stdout)
if allTargets:
print("%d tag(s) are defined: "%(len(init.tags())), file=stdout)
for t in sorted([' %-15s (%d targets)'%(t, len(init.tags()[t])) for t in init.tags()]):
print(t, file=stdout)
elif task == _TASK_LIST_TARGET_INFO:
if findTargetsList == '*': findTargetsList = init.targets().values()
for t in sorted(findTargetsList, key=lambda t:(t.type+' '+t.name)):
print('- %s priority: %s, tags: %s, location: \n %s'%(t, t.getPriority(), t.getTags(), t.location), file=stdout)
elif task == _TASK_LIST_FIND_TARGETS:
# sort matches at start of path first, then anywhere in name, finally anywhere in type
for t in findTargetsList:
# this must be very easy to copy+paste, so don't put anything else on the line at all
print('%s'%(t.name), file=stdout)
elif task == _TASK_LIST_SEARCH:
def showPatternMatches(x): # case sensitive is fine (and probably useful)
if searchPattern.replace('\\', '/') in x.replace('\\','/'): return True # basic substring check (with path normalization)
if '*' in searchPattern or '?' in searchPattern or '[' in searchPattern: # probably a regex
if re.search(searchPattern, x): return True
return False
for t in init.targets().values():
t._resolveTargetPath(init)
print('', file=stdout)
tagMatches = [t for t in init.tags() if showPatternMatches(t)]
if tagMatches:
print ('%d matching tags:'%len(tagMatches), file=stdout)
for t in sorted(tagMatches):
print(t, file=stdout)
print('', file=stdout)
targetMatches = [t for t in init.targets().values() if showPatternMatches(t.name) or showPatternMatches(t.path)]
if targetMatches:
print ('%d matching targets:'%len(targetMatches), file=stdout)
for t in sorted(targetMatches, key=lambda t:(t.type+' '+t.name)):
print('- %s priority: %s, tags: [%s]\n output: %s\n defined: %s'%(t, t.getPriority(), ' '.join(sorted(t.getTags())) or 'none', os.path.relpath(t.path), t.location), file=stdout)
print('', file=stdout)
propMatches = {key:value for (key,value) in init.getProperties().items() if showPatternMatches(key)}
if propMatches:
print('%d matching properties:'%len(propMatches), file=stdout)
pad = max(list(map(len, propMatches.keys())))
for k in sorted(propMatches.keys()):
print(('%'+str(pad)+'s = %s') % (k, propMatches[k]), file=stdout)
if init._propertyLocations[k]: # don't do this for built-in property like BUILD_MODE
print(('%'+str(pad)+'s (defined: %s)') % ('', init._propertyLocations[k]), file=stdout)
options = init.mergeOptions(None)
optionMatches = {key:value for (key,value) in options.items() if showPatternMatches(key)}
if optionMatches:
print('%d matching options:'%len(optionMatches), file=stdout)
pad = max(list(map(len, optionMatches.keys())))
for k in sorted(optionMatches.keys()):
print(('%'+str(pad)+'s = %s') % (k, optionMatches[k]), file=stdout)
elif task in [_TASK_BUILD, _TASK_CLEAN, _TASK_REBUILD]:
if not logFile:
if allTargets:
buildtag = None
else:
buildtag = 'custom'
logFile = _maybeCustomizeLogFilename(init.getPropertyValue('LOG_FILE'),
buildtag,
task==_TASK_CLEAN)
logFile = os.path.abspath(logFile)
logdir = os.path.dirname(logFile)
if logdir and not os.path.exists(logdir): mkdir(logdir)
log.critical('Writing build log to: %s', os.path.abspath(logFile))
# also buffer the .log file, since it's just a lot harder to read when multiple target lines are all jumbled up;
# we have an undocumented env var for disabling this in case of debugging
if os.getenv('XPYBUILD_LOGFILE_OUTPUT_BUFFERING_DISABLED','')=='true': outputBufferingDisabled = True
logFileStream = OutputBufferingStreamWrapper(open(logFile, 'w', encoding='UTF-8'), bufferingDisabled=outputBufferingDisabled)
hdlr = logging.StreamHandler(logFileStream)
hdlr.setFormatter(logging.Formatter('%(asctime)s %(relativeCreated)05d %(levelname)-8s [%(threadName)s %(thread)5d] %(name)-10s - %(message)s', None))
hdlr.setLevel(logLevel or logging.INFO)
logging.getLogger().addHandler(hdlr)
log.info('Using xpybuild %s from %s on Python %s.%s.%s', XPYBUILD_VERSION, os.path.normpath(os.path.dirname(__file__)), sys.version_info[0], sys.version_info[1], sys.version_info[2])
log.info('Using build options: %s (logfile target outputBuffering=%s, stdout target outputBuffering=%s)', buildOptions, not outputBufferingDisabled, not wrapper.bufferingDisabled)
try:
# sometimes useful to have this info available
import socket, getpass
log.info('Build running on %s as user %s', socket.gethostname(), getpass.getuser())
except Exception as e:
log.info('Failed to get host/user: %s', e)
log.info('Default encoding for subprocesses assumed to be: %s (stdout=%s, preferred=%s)',
DEFAULT_PROCESS_ENCODING, stdout.encoding, locale.getpreferredencoding())
def lowerCurrentProcessPriority():
if xpybuild.buildcommon.IS_WINDOWS:
import win32process, win32api,win32con
win32process.SetPriorityClass(win32api.GetCurrentProcess(), win32process.BELOW_NORMAL_PRIORITY_CLASS)
else:
# on unix, people may run nice before executing the process, so
# only change the priority unilaterally if it's currently at its
# default value
if os.nice(0) == 0:
os.nice(1) # change to 1 below the current level
try:
# if possible, set priority of builds to below normal by default,
# to avoid starving machines (e.g. on windows) of resources
# that should be used for interactive processes
if os.getenv('XPYBUILD_DISABLE_PRIORITY_CHANGE','') != 'true':
lowerCurrentProcessPriority()
log.info('Successfully changed process priority to below normal')
except Exception as e:
log.warning('Failed to lower current process priority: %s'%e)
if buildOptions['ignore-deps']:
log.warning('The ignore-deps option is enabled: dependency graph will be ignored for all targets that already exist on disk, so correctness is not guaranteed')
for (k,v) in sorted(init.getProperties().items()):
log.info('Setting property %s=%s', k, v)
try:
DATE_TIME_FORMAT = "%a %Y-%m-%d %H:%M:%S %Z"
errorsList = []
if task in [_TASK_CLEAN, _TASK_REBUILD]:
startTime = time.time()
log.critical('Starting "%s" clean "%s" at %s', init.getPropertyValue('BUILD_MODE'), init.getPropertyValue('BUILD_NUMBER'),
time.strftime(DATE_TIME_FORMAT, time.localtime( startTime )))
cleanBuildOptions = buildOptions.copy()
cleanBuildOptions['clean'] = True
if allTargets: cleanBuildOptions['ignore-deps'] = True
scheduler = BuildScheduler(init, selectedTargets, cleanBuildOptions)
errorsList, targetsBuilt, targetsCompleted, totalTargets = scheduler.run()
if allTargets and not cleanBuildOptions['dry-run']: # special-case this common case
for dir in init.getOutputDirs():
deleteDir(dir)
log.critical('Completed "%s" clean "%s" at %s after %s\n', init.getPropertyValue('BUILD_MODE'), init.getPropertyValue('BUILD_NUMBER'),
time.strftime(DATE_TIME_FORMAT, time.localtime( startTime )), formatTimePeriod(time.time()-startTime))
if errorsList:
log.critical('XPYBUILD FAILED: %d error(s): \n %s', len(errorsList), '\n '.join(sorted(errorsList)))
return 3
if task == _TASK_REBUILD:
# we must reload the build file here, as it's the only way of flushing out
# cached data (especially in PathSets) that may have changed as a
# result of the clean
init = loadBuildFile()
if task in [_TASK_BUILD, _TASK_REBUILD] and not errorsList:
for cb in init.getPreBuildCallbacks():
try:
cb(BuildContext(init))
except BuildException as be:
log.error("Pre-build check failed: %s", be)
return 7
buildtype = 'incremental' if any(os.path.exists(dir) for dir in init.getOutputDirs()) else 'fresh'
if not buildOptions['dry-run']:
for dir in init.getOutputDirs():
log.info('Creating output directory: %s', dir)
mkdir(dir)
startTime = time.time()
log.critical('Starting %s "%s" build "%s" at %s using %d workers', buildtype,
init.getPropertyValue('BUILD_MODE'), init.getPropertyValue('BUILD_NUMBER'),
time.strftime(DATE_TIME_FORMAT, time.localtime( startTime )),
buildOptions['workers']
)
buildOptions['clean'] = False
scheduler = BuildScheduler(init, selectedTargets, buildOptions)
errorsList, targetsBuilt, targetsCompleted, totalTargets = scheduler.run()
log.critical('Completed %s "%s" build "%s" at %s after %s\n', buildtype, init.getPropertyValue('BUILD_MODE'), init.getPropertyValue('BUILD_NUMBER'),
time.strftime(DATE_TIME_FORMAT, time.localtime( startTime )), formatTimePeriod(time.time()-startTime))
if 'timeFile' in buildOptions:
logTargetTimes(buildOptions['timeFile'], scheduler, init)
if errorsList:
# heuristically: it's useful to have them in order of failure when a small number, but if there are
# lots then it's too hard to read and better to sort, so similar ones are together
if len(errorsList)>=10: errorsList.sort()
log.critical('*** XPYBUILD FAILED: %d error(s) (aborted with %d targets outstanding): \n %s', len(errorsList), totalTargets-targetsCompleted, '\n '.join(errorsList))
return 4
else:
# using *** here means we get a valid final progress message
log.critical('*** XPYBUILD SUCCEEDED: %s built (%d up-to-date)', targetsBuilt if targetsBuilt else '<NO TARGETS>', (totalTargets-targetsBuilt))
return 0
finally:
publishArtifact('Xpybuild logfile', logFile)
else:
raise Exception('Task type not implemented yet - '+task) # should not happen
except BuildException as e:
# hopefully we don't end up here very often
log.error('*** XPYBUILD FAILED: %s', e.toMultiLineString(None))
return 5
except Exception as e:
log.exception('*** XPYBUILD FAILED: ')
return 6
def javac(output, inputs, classpath, options, logbasename, targetname, workDir):
""" Compile some java files to class files.
Will raise BuildException if compilation fails.
@param output: path to a directory in which to put the class files (will be created)
@param inputs: list of paths (.java files) to be compiled
@param classpath: classpath to compile with, as a string
@param options: options map. javac.options is a list of additional arguments, javac.source is the source version,
javac.target is the target version
@param logbasename: absolute, expanded, path to a directory and filename prefix
to use for files such as .err, .out, etc files
@param targetname: to log appropriate error messages
@param workDir: where temporary files are stored.
"""
assert logbasename and '$' not in logbasename
logbasename = os.path.normpath(logbasename)
# make the output directory
if not os.path.exists(output): mkdir(output)
# location of javac
if options['java.home']:
javacpath = os.path.join(options['java.home'], "bin/javac")
else:
javacpath = "javac" # just get it from the path
# store the list of files in a temporary file, then build from that.
mkdir(workDir)
argsfile = os.path.join(workDir, "javac_args.txt")
# build up the arguments
args = ["-d", output]
if options["javac.source"]: args.extend(["-source", options["javac.source"]])
if options["javac.target"]: args.extend(["-target", options["javac.target"]])
if options["javac.encoding"]: args.extend(["-encoding", options["javac.encoding"]])
if options["javac.debug"]:
args.append('-g')
if options['javac.warningsAsErrors']:
args.append('-Werror')
# TODO: should add -Xlint options here I think
args.extend(getStringList(options['javac.options']))
if classpath: args.extend(['-cp', classpath])
args.extend([x for x in inputs if x.endswith('.java')]) # automatically filter out non-java files
with openForWrite(argsfile, 'w', encoding=locale.getpreferredencoding()) as f:
for a in args:
f.write('"%s"'%a.replace('\\','\\\\')+'\n')
success=False
try:
log.info('Executing javac for %s, writing output to %s: %s', targetname, logbasename+'.out', ''.join(['\n\t"%s"'%x for x in [javacpath]+args]))
# make sure we have no old ones hanging around still
try:
deleteFile(logbasename+'-errors.txt', allowRetry=True)
deleteFile(logbasename+'-warnings.txt', allowRetry=True)
deleteFile(logbasename+'.out', allowRetry=True)
except Exception as e:
log.info('Cleaning up file failed: %s' % e)
outputHandler = options.get('javac.outputHandlerFactory', JavacProcessOutputHandler)(targetname, options=options)
if hasattr(outputHandler, 'setJavacLogBasename'):
outputHandler.setJavacLogBasename(logbasename)
call([javacpath, "@%s" % argsfile], outputHandler=outputHandler, outputEncoding='UTF-8', cwd=output, timeout=options['process.timeout'])
if (not os.listdir(output)): # unlikely, but useful failsafe
raise EnvironmentError('javac command failed to create any target files (but returned no error code); see output at "%s"'%(logbasename+'.out'))
success = True
finally:
if not success and classpath:
log.info('Classpath for failed javac was: \n %s', '\n '.join(classpath.split(os.pathsep)))
def run(self, context):
options = self.options
# make sure temp dir exists
mkdir(self.workDir)
classes = os.path.join(self.workDir, "classes") # output dir for classes
# create the classpath, sorting within PathSet (for determinism), but retaining original order of
# PathSet elements in the list
classpath = os.pathsep.join(self.classpath.resolve(context))
# compile everything
mkdir(classes) # (need this for assembling other files to package later on, even if we don't do any javac)
if self.compile:
mkdir(self.getOption('javac.logs'))
javac(classes, self.compile.resolve(context), classpath, options=options, logbasename=options.get('javac.logs')+'/'+targetNameToUniqueId(self.name), targetname=self.name, workDir=self.workDir)
manifest = os.path.join(self.workDir, "MANIFEST.MF") # manifest file
if isinstance(self.manifest, str):
manifest = context.getFullPath(self.manifest, self.baseDir)
elif self.manifest == None:
manifest = None
else: # generate one
# rewrite property values in the manifest
manifest_entries = {}
for i in self.manifest:
manifest_entries[i] = context.expandPropertyValues(self.manifest[i])
# determine classpath for manifest
classpath_entries = []
if "Class-path" not in manifest_entries: # assuming it wasn't hardcoded, set it here
for src, dest in self.classpath.resolveWithDestinations(context):
# we definitely do want to support use of ".." in destinations here, it can be very useful
classpath_entries.append(dest)
assert isinstance(options['jar.manifest.classpathAppend'], list), options['jar.manifest.classpathAppend'] # must not be a string
classpath_entries.extend(options['jar.manifest.classpathAppend'] or [])
# need to always use / not \ for these to be valid
classpath_entries = [p.replace(os.path.sep, '/').replace('\\', '/') for p in classpath_entries if p]
if classpath_entries:
manifest_entries["Class-path"] = " ".join(classpath_entries) # include the classpath from here
if not manifest_entries.get('Class-path'): # suppress this element entirely if not needed, otherwise there would be no way to have an empty classpath
manifest_entries.pop('Class-path','')
# create the manifest file
create_manifest(manifest, manifest_entries, options=options)
# copy in the additional things to include
for (src, dest) in self.package.resolveWithDestinations(context):
if '..' in dest: raise Exception('This target does not permit packaged destination paths to contain ".." relative path expressions')
mkdir(os.path.dirname(os.path.join(classes, dest)))
destpath = normLongPath(classes+'/'+dest)
srcpath = normLongPath(src)
if os.path.isdir(srcpath):
mkdir(destpath)
else:
with open(srcpath, 'rb') as s:
with openForWrite(destpath, 'wb') as d:
d.write(s.read())
# create the jar
jar(self.path, manifest, classes, options=options, preserveManifestFormatting=self.preserveManifestFormatting,
outputHandler=ProcessOutputHandler.create('jar', treatStdErrAsErrors=False,options=options))
def run(self, context):
self.keystore = context.expandPropertyValues(self.keystore)
options = self.options
mkdir(self.path)
for src, dest in self.jars.resolveWithDestinations(context):
if '..' in dest:
# to avoid people abusing this to copy files outside the dest directory!
raise Exception('This target does not permit destination paths to contain ".." relative path expressions')
try:
with open(src, 'rb') as s:
with openForWrite(os.path.join(self.path, dest), 'wb') as d:
d.write(s.read())
shutil.copystat(src, os.path.join(self.path, dest))
# When we re-jar with the user specified manifest entries, jar will complain
# about duplicate attributes IF the original MANIFEST.MF already has those entries.
# This is happening for latest version of SL where Application-Name, Permission etc
# were already there.
#
# The block of code below will first extract the original MANIFEST.MF from the source
# jar file, read all manifest entry to a list. When constructing the new manifest entries,
# make sure the old MANIFEST.MF doesn't have that entry before putting the new manifest entry
# to the list. This will avoid the duplicate attribute error.
#
if self.manifestDefaults:
lines = []
# read each line of MANIFEST.MF of the original jar and put them in lines
with zipfile.ZipFile(src, 'r') as zf:
lst = zf.infolist()
for zi in lst:
fn = zi.filename
if fn.lower().endswith('manifest.mf'):
try:
manifest_txt = zf.read(zi.filename).decode('utf-8', errors='strict')
except Exception as e:
raise BuildException('Failed reading the manifest file %s with exception:%s' % (fn, e))
# if we have all manifest text, parse and save each line
if manifest_txt:
# CR LF | LF | CR can be there as line feed and hence the code below
lines = manifest_txt.replace('\r\n', '\n').replace('\r','\n').split('\n')
# done
break
original_entries = collections.OrderedDict() # to ensure we don't overwrite/duplicate these
# populate the manifest_entries with original values from original manifest
for l in lines:
if ':' in l and not l.startswith(' '): # ignore continuation lines etc because keys are all we care about
key,value = l.split(':', 1)
original_entries[key] = value.strip()
# build up a list of the new manifest entries (will be merged into any existing manifest by jar)
manifest_entries = collections.OrderedDict()
for i in self.manifestDefaults:
# if entry isn't there yet, add to the list
if i not in original_entries:
manifest_entries[i] = context.expandPropertyValues(self.manifestDefaults[i])
# create the manifest file
# we want to add the manifest entries explicitly specified here but
# NOT the 'default' manifest entries we usually add, since these
# are likely to have been set already, and we do not want duplicates
mkdir(self.workDir)
manifest = os.path.join(self.workDir, "MANIFEST.MF") # manifest file
options = dict(options)
options['jar.manifest.defaults'] = {}
create_manifest(manifest, manifest_entries, options)
# update the EXISTING jar file with the new manifest entries, which will be merged into
# existing manifest by the jar tool
jar(os.path.join(self.path, dest), manifest, None, options, update=True)
signjar(os.path.join(self.path, dest), self.keystore, options, alias=self.alias, storepass=self.storepass,
outputHandler=ProcessOutputHandler.create('signjars', treatStdErrAsErrors=False, options=options))
except BuildException as e:
raise BuildException('Error processing %s: %s'%(os.path.basename(dest), e))
def run(self, context):
mkdir(os.path.dirname(self.path))
uri = context.expandPropertyValues(self.uri)
urllib.request.urlretrieve(uri, self.path)
def run(self, context):
self.log.info("Copying %s to %s", self.src, self.path)
src = self.src.resolveWithDestinations(
context) # a map of srcAbsolute: destRelative
symlinks = self.options['Copy.symlinks']
if isinstance(symlinks, str): symlinks = symlinks.lower() == 'true'
assert symlinks in [True, False], repr(symlinks)
# implicitly ensure parent of target exists, to keep things simple
copied = 0
if not isDirPath(self.name):
# it's a simple file operation.
if len(src) != 1:
raise BuildException(
'Copy destination must be a directory (ending with "/") when multiple sources are specified (not: %s)'
% src)
src, mappedDest = src[0]
if isDirPath(src):
raise BuildException(
'Copy source must be files (or PathSets) not directories: %s'
% src)
mkdir(os.path.dirname(self.path))
self._copyFile(
context, src, self.path
) # we kindof have to ignore mappedDest here, since the target path already fully defines it
if self.mode:
os.chmod(self.path, self.mode)
copied += 1
else:
lastDirCreated = None
for (srcAbs, destRel) in src:
srcAbs = normLongPath(srcAbs)
dest = normLongPath(self.path + destRel)
# there should not be any directories here only files from pathsets
if '..' in destRel:
# to avoid people abusing this to copy files outside the dest directory!
raise Exception(
'This target does not permit destination paths to contain ".." relative path expressions'
)
issymlink = symlinks and os.path.islink(srcAbs.rstrip(os.sep))
if isDirPath(
srcAbs
) and not issymlink: # allows creating of empty directories.
mkdir(dest)
else:
#self.log.debug('Processing %s -> %s i.e. %s', srcAbs, destRel, dest)
if issymlink: # this may be a directory path, and dirname will fail unless we strip off the /
dest = dest.rstrip(os.sep)
if not lastDirCreated or lastDirCreated != os.path.dirname(
dest):
lastDirCreated = os.path.dirname(dest)
self.log.debug('Creating intermediate dir %s',
lastDirCreated)
mkdir(lastDirCreated)
try:
if issymlink:
os.symlink(os.readlink(srcAbs.rstrip(os.sep)),
dest)
else:
self._copyFile(context, srcAbs, dest)
if self.mode:
os.chmod(dest, self.mode)
except Exception as e:
raise BuildException(
'Error copying from "%s" to "%s"' % (srcAbs, dest),
causedBy=True)
copied += 1
self.log.info('Copied %d file(s) to %s', copied, self.path)
def run(self, context):
if self.cwd: self.cwd = context.getFullPath(self.cwd, self.baseDir)
if isDirPath(self.path):
mkdir(self.path)
cwd = self.cwd or self.path
else:
mkdir(os.path.dirname(self.path))
cwd = self.cwd or self.workDir
mkdir(self.workDir)
commands = self._resolveCommands(context)
assert len(
commands) > 0, 'No commands were specified to run in this target!'
if len(commands) > 1:
assert not (
self.redirectStdOutToTarget or self.stdout or self.stderr
), 'Invalid argument was specified for multiple commands mode'
cmdindex = 0
for cmd in commands:
cmdindex += 1
# this location is a lot easier to find than the target's workdir
logbasename = os.path.normpath(
context.getPropertyValue('BUILD_WORK_DIR') +
'/CustomCommandOutput/' + os.path.basename(cmd[0]) + "." +
targetNameToUniqueId(self.name))
if cmdindex > 1:
logbasename = logbasename + ".%d" % cmdindex # make this unique
cmdDisplaySuffix = ' #%d' % (cmdindex) if len(commands) > 1 else ''
stdoutPath = context.getFullPath(
self.path if self.redirectStdOutToTarget else
(self.stdout or logbasename + '.out'),
defaultDir='${BUILD_WORK_DIR}/CustomCommandOutput/')
stderrPath = context.getFullPath(
self.stderr or logbasename + '.err',
defaultDir='${BUILD_WORK_DIR}/CustomCommandOutput/')
self.log.info('Building %s by executing command%s: %s', self.name,
cmdDisplaySuffix,
''.join(['\n\t"%s"' % x for x in cmd]))
if self.cwd and cmdindex == 1:
self.log.info(' building %s from working directory: %s',
self.name, self.cwd) # only print if overridden
env = self.env or {}
if env:
if callable(env):
env = env(context)
else:
env = {
k: None if None == env[k] else self._resolveItem(
env[k], context)
for k in env
}
if cmdindex == 1:
self.log.info(
' environment overrides for %s are: %s', self.name,
''.join(['\n\t"%s=%s"' % (k, env[k]) for k in env]))
for k in os.environ:
if k not in env: env[k] = os.getenv(k)
for k in list(env.keys()):
if None == env[k]:
del env[k]
self.log.info(
' output from %s will be written to "%s" and "%s"',
self.name + cmdDisplaySuffix, stdoutPath, stderrPath)
if not os.path.exists(cmd[0]) and not (
IS_WINDOWS and os.path.exists(cmd[0] + '.exe')):
raise BuildException(
'Cannot run command because the executable does not exist: "%s"'
% (cmd[0]),
location=self.location)
encoding = self.options['common.processOutputEncodingDecider'](
context, cmd[0])
handler = self.options['CustomCommand.outputHandlerFactory']
if handler: # create a new handler for each command
handler = handler(str(self), options=self.options)
success = False
rc = None
try:
# maybe send output to a file instead
mkdir(os.path.dirname(logbasename))
with open(
stderrPath,
'wb') as fe: # can't use openForWrite with subprocess
with open(stdoutPath, 'wb') as fo:
process = subprocess.Popen(cmd,
stderr=fe,
stdout=fo,
cwd=cwd,
env=env)
rc = _wait_with_timeout(
process,
'%s(%s)' % (self.name, os.path.basename(cmd[0])),
self.options['process.timeout'], False)
success = rc == 0
finally:
try:
if os.path.getsize(stderrPath) == 0 and not self.stderr:
deleteFile(stderrPath, allowRetry=True)
if not self.redirectStdOutToTarget and os.path.getsize(
stdoutPath) == 0 and not self.stdout:
deleteFile(stdoutPath, allowRetry=True)
except Exception as e:
# stupid windows, it passes understanding
self.log.info(
'Failed to delete empty .out/.err files (ignoring error as it is not critical): %s',
e)
#if not os.listdir(self.workDir): deleteDir(self.workDir) # don't leave empty work dirs around
mainlog = '<command did not write any stdout/stderr>'
logMethod = self.log.info if success else self.log.error
if (handler
or not self.redirectStdOutToTarget) and os.path.isfile(
stdoutPath) and os.path.getsize(stdoutPath) > 0:
if handler:
with open(stdoutPath,
'r',
encoding=encoding,
errors='replace') as f:
for l in f:
handler.handleLine(l, isstderr=False)
elif os.path.getsize(stdoutPath) < 15 * 1024:
logMethod(
' stdout from %s is: \n%s',
self.name + cmdDisplaySuffix,
open(stdoutPath,
'r',
encoding=encoding,
errors='replace').read().replace(
'\n', '\n\t'))
mainlog = stdoutPath
if not success:
context.publishArtifact(
'%s%s stdout' % (self, cmdDisplaySuffix),
stdoutPath)
if os.path.isfile(
stderrPath) and os.path.getsize(stderrPath) > 0:
if handler:
with open(stderrPath,
'r',
encoding=encoding,
errors='replace') as f:
for l in f:
handler.handleLine(l, isstderr=True)
elif os.path.getsize(stderrPath) < 15 * 1024:
logMethod(
' stderr from %s is: \n%s',
self.name + cmdDisplaySuffix,
open(stderrPath,
'r',
encoding=encoding,
errors='replace').read().replace(
'\n', '\n\t'))
mainlog = stderrPath # take precedence over stdout
if not success:
context.publishArtifact(
'%s%s stderr' % (self, cmdDisplaySuffix),
stderrPath)
if handler:
handler.handleEnd(returnCode=rc)
elif rc != None and rc != 0 and not handler:
if IS_WINDOWS:
quotearg = lambda c: '"%s"' % c if ' ' in c else c
else:
quotearg = shlex.quote
# having it in this format makes it easier for people to re-run the command manually
self.log.info(' full command line is: %s',
' '.join(quotearg(c) for c in cmd))
raise BuildException(
'%s command%s failed with error code %s; see output at "%s" or look under %s'
% (os.path.basename(
cmd[0]), cmdDisplaySuffix, rc, mainlog, cwd),
location=self.location)
# final sanity check
if not os.path.exists(self.path):
raise BuildException(
'%s returned no error code but did not create the output file/dir; see output at "%s" or look under %s'
% (self, mainlog, cwd),
location=self.location)
if (not isDirPath(self.path)) and (not os.path.isfile(self.path)):
raise BuildException(
'%s did not create a file as expected (please check that trailing "/" is used if and only if a directory output is intended)'
% self,
location=self.location)
if isDirPath(self.path) and not os.listdir(self.path):
raise BuildException('%s created an empty directory' % self,
location=self.location)