def __init__(self, jar, compile, classpath, manifest, options=None, package=None, preserveManifestFormatting=False): """ @param jar: path to jar to create @param compile: PathSet (or list) of things to compile @param classpath: PathSet (or list) of things to be on the classpath; destination mapping indicates how they will appear in the manifest @param manifest: map of manifest entries, OR a string with the filename to use OR None to disable manifest generation and just produce a normal zip @param options: generic target options map @param package: PathSet (or list) of other files to include in the jar; destination mapping indicates where they will appear in the jar @param preserveManifestFormatting: an advanced option that prevents the jar tool from reformatting the specified manifest file to comply with Java conventions (also prevents manifest merging if jar already exists) """ self.compile = FilteredPathSet(_isJavaFile, PathSet(compile)) if compile else None self.classpath = PathSet(classpath) self.package = PathSet(package) self.manifest = manifest BaseTarget.__init__(self, jar, [self.compile,self.classpath,self.package, manifest if isinstance(manifest, basestring) else None]) self.options = options self.preserveManifestFormatting = preserveManifestFormatting
class Javadoc(BaseTarget):
""" Creates javadoc from a set of input files
"""
def __init__(self, destdir, source, classpath, options):
"""
@param destdir: directory to create docs in
@param source: a set of files to build from
@param classpath: a list of jars needed for the classpath
@param options: javadoc.-prefixed options map
"""
self.sources = PathSet(source)
self.classpath = PathSet(classpath)
BaseTarget.__init__(self, destdir, [self.sources, self.classpath])
self.options = options
def run(self, context):
options = context.mergeOptions(self) # get the merged options
classpath = os.pathsep.join(self.classpath.resolve(context))
javadoc(self.path, self.sources.resolve(context), classpath, options,
outputHandler=ProcessOutputHandler('javadoc', treatStdErrAsErrors=False, options=options))
def getHashableImplicitInputs(self, context):
# changes in the manifest text should cause a rebuild
# for now, don't bother factoring global jar.manifest.defaults option
# in here (it'll almost never change anyway)
return super(Javadoc, self).getHashableImplicitInputs(context) + \
sorted(['option: %s = "%s"'%(k,v) for (k,v) in context.mergeOptions(self).items()
if k and k.startswith('javadoc.')])
def __init__(self, bin, objects, libs=None, libpaths=None, shared=False, options=None, flags=None, dependencies=None):
"""
@param bin: the output binary
@param objects: a (list of) input object
@param libs: a (list of) libraries linked against (optional) in platform-neutral format.
Can include list properties like '${FOO_LIB_NAMES[]}'.
@param libpaths: a (list of) additional library search directories (optional)
@param shared: if true compiles to a shared object (.dll or .so) (optional, defaults to false)
@param flags: a list of additional linker flags
@param options: a map of options to the underlying operation specific to this target (optional)
@param dependencies: a list of additional dependencies (targets or files)
"""
self.objects = PathSet(objects)
self.libs = libs or []
self.libpaths = PathSet(libpaths or [])
self.shared=shared
self.options = options
self.flags = flags or []
BaseTarget.__init__(self, bin, PathSet(self.objects, (dependencies or [])))
self.tags('native')
class Link(BaseTarget):
""" A target that links object files to binaries
"""
def __init__(self, bin, objects, libs=None, libpaths=None, shared=False, options=None, flags=None, dependencies=None):
"""
@param bin: the output binary
@param objects: a (list of) input object
@param libs: a (list of) libraries linked against (optional) in platform-neutral format.
Can include list properties like '${FOO_LIB_NAMES[]}'.
@param libpaths: a (list of) additional library search directories (optional)
@param shared: if true compiles to a shared object (.dll or .so) (optional, defaults to false)
@param flags: a list of additional linker flags
@param options: a map of options to the underlying operation specific to this target (optional)
@param dependencies: a list of additional dependencies (targets or files)
"""
self.objects = PathSet(objects)
self.libs = libs or []
self.libpaths = PathSet(libpaths or [])
self.shared=shared
self.options = options
self.flags = flags or []
BaseTarget.__init__(self, bin, PathSet(self.objects, (dependencies or [])))
self.tags('native')
def run(self, context):
options = context.mergeOptions(self) # get the merged options
mkdir(os.path.dirname(self.path))
options['native.compilers'].linker.link(context, output=self.path,
options=options,
flags=options['native.link.flags']+self.flags,
shared=self.shared,
src=self.objects.resolve(context),
libs=flatten([map(string.strip, context.expandPropertyValues(x, expandList=True)) for x in self.libs+options['native.libs'] if x]),
libdirs=flatten(self.libpaths.resolve(context)+[context.expandPropertyValues(x, expandList=True) for x in options['native.libpaths']]))
def getHashableImplicitInputs(self, context):
r = super(Link, self).getHashableImplicitInputs(context)
options = context.mergeOptions(self)
r.append('libs: '+context.expandPropertyValues(str(self.libs+options['native.libs'])))
r.append('libpaths: '+context.expandPropertyValues(str(self.libpaths)))
r.append('native.libpaths: %s'%options['native.libpaths'])
r.append('shared: %s, flags=%s'%(self.shared, self.flags))
return r
def __init__(self, destdir, source, classpath, options): """ @param destdir: directory to create docs in @param source: a set of files to build from @param classpath: a list of jars needed for the classpath @param options: javadoc.-prefixed options map """ self.sources = PathSet(source) self.classpath = PathSet(classpath) BaseTarget.__init__(self, destdir, [self.sources, self.classpath]) self.options = options
def __init__(self, object, source, includes=None, flags=None, options=None, dependencies=None):
"""
@param object: the object file to generate
@param source: a (list of) source files
@param includes: a (list of) include directories
@param flags: a list of additional compiler flags
@param dependencies: a list of additional dependencies that need to be built
before this target
"""
self.source = PathSet(source)
self.includes = PathSet(includes or [])
self.flags = flags or []
self.makedepend = CompilerMakeDependsPathSet(self, self.source, flags=self.flags, includes=self.includes)
BaseTarget.__init__(self, object, [dependencies or [], self.makedepend])
self.options = options or {}
self.tags('native')
class Ar(BaseTarget):
""" A target that compiles .a files from collections of .o files
"""
def __init__(self, bin, objects):
"""
@param bin: the output library
@param objects: a (list of) input objects
"""
self.objects = PathSet(objects)
BaseTarget.__init__(self, bin, self.objects)
self.tags('native')
def run(self, context):
options = context.mergeOptions(self) # get the merged options
mkdir(os.path.dirname(self.path))
options['native.compilers'].archiver.archive(context, output=self.path,
options=options,
src=self.objects.resolve(context))
def getHashableImplicitInputs(self, context):
r = super(Ar, self).getHashableImplicitInputs(context)
r.append('objects: %s'%self.objects)
return r
class Tarball(BaseTarget):
""" A target that creates a zip archive from a set of input files.
"""
def __init__(self, archive, inputs):
"""
archive: the archive to be created
inputs: the files (usually pathsets) to be included in the archive.
"""
self.inputs = PathSet(inputs)
BaseTarget.__init__(self, archive, self.inputs)
def run(self, context):
mkdir(os.path.dirname(self.path))
with tarfile.open(normLongPath(self.path), 'w:gz') as output:
for (f, o) in self.inputs.resolveWithDestinations(context):
output.add(normLongPath(f).rstrip('/\\'), o)
def getHashableImplicitInputs(self, context):
# TODO: move to BaseTarget
r = super(Tarball, self).getHashableImplicitInputs(context)
# include source representation of deps list, so that changes to the list get reflected
# this way of doing property expansion on the repr is a convenient
# shortcut (we want to expand property values to detect changes in
# versions etc that should trigger a rebuild, but just not do any
# globbing/searches here)
r.append('src: '+context.expandPropertyValues(('%s'%self.inputs)))
return r
def __init__(self, archive, inputs): """ archive: the archive to be created inputs: the files (usually pathsets) to be included in the archive. """ self.inputs = PathSet(inputs) BaseTarget.__init__(self, archive, self.inputs)
def __init__(self, bin, objects):
"""
@param bin: the output library
@param objects: a (list of) input objects
"""
self.objects = PathSet(objects)
BaseTarget.__init__(self, bin, self.objects)
self.tags('native')
class C(BaseTarget):
""" A target that compiles a C source file to a .o
"""
def __init__(self, object, source, includes=None, flags=None, options=None, dependencies=None):
"""
@param object: the object file to generate
@param source: a (list of) source files
@param includes: a (list of) include directories
@param flags: a list of additional compiler flags
@param dependencies: a list of additional dependencies that need to be built
before this target
"""
self.source = PathSet(source)
self.includes = PathSet(includes or [])
self.flags = flags or []
self.makedepend = CompilerMakeDependsPathSet(self, self.source, flags=self.flags, includes=self.includes)
BaseTarget.__init__(self, object, [dependencies or [], self.makedepend])
self.options = options or {}
self.tags('native')
def run(self, context):
options = context.mergeOptions(self) # get the merged options
mkdir(os.path.dirname(self.path))
options['native.compilers'].ccompiler.compile(context, output=self.path,
options=options,
flags=flatten((options['native.c.flags'] or options['native.cxx.flags'])+[context.expandPropertyValues(x).split(' ') for x in self.flags]),
src=self.source.resolve(context),
includes=flatten(self.includes.resolve(context)+[context.expandPropertyValues(x, expandList=True) for x in options['native.include']]))
def clean(self, context):
self.makedepend.clean()
BaseTarget.clean(self, context)
def getHashableImplicitInputs(self, context):
r = super(C, self).getHashableImplicitInputs(context)
# include input to makedepends, since even without running makedepends
# we know we're out of date if inputs have changed
r.append(context.expandPropertyValues(str(self.makedepend)))
return r
class Zip(BaseTarget):
""" A target that creates a zip archive from a set of input files.
"""
def __init__(self, archive, inputs):
"""
archive: the archive to be created
inputs: the files (usually pathsets) to be included in the archive.
"""
self.inputs = PathSet(inputs)
BaseTarget.__init__(self, archive, self.inputs)
def run(self, context):
mkdir(os.path.dirname(self.path))
alreadyDone = set()
with zipfile.ZipFile(normLongPath(self.path), 'w') as output:
for (f, o) in self.inputs.resolveWithDestinations(context):
# if we don't check for duplicate entries we'll end up creating an invalid zip
if o in alreadyDone:
dupsrc = ['"%s"'%src for (src, dest) in self.inputs.resolveWithDestinations(context) if dest == o]
raise BuildException('Duplicate zip entry "%s" from: %s'%(o, ', '.join(dupsrc)))
alreadyDone.add(o)
# can't compress directory entries! (it messes up Java)
output.write(normLongPath(f).rstrip('/\\'), o, zipfile.ZIP_STORED if isDirPath(f) else zipfile.ZIP_DEFLATED)
def getHashableImplicitInputs(self, context):
# TODO: move to BaseTarget
r = super(Zip, self).getHashableImplicitInputs(context)
# include source representation of deps list, so that changes to the list get reflected
# this way of doing property expansion on the repr is a convenient
# shortcut (we want to expand property values to detect changes in
# versions etc that should trigger a rebuild, but just not do any
# globbing/searches here)
r.append('src: '+context.expandPropertyValues(('%s'%self.inputs)))
return r
def __init__(self, imagename, mode, inputs, depimage=None, dockerfile=None, buildArgs=None):
"""
imagename: the name/tag of the image to build
"""
self.imagename = imagename
self.depimage = depimage
self.mode = mode
self.dockerfile = dockerfile
self.buildArgs = buildArgs
self.stampfile = '${BUILD_WORK_DIR}/targets/docker/.%s' % re.sub(r'[\\/]', '_', imagename)
self.depstampfile = '${BUILD_WORK_DIR}/targets/docker/.%s' % re.sub(r'[\\/]', '_', depimage) if depimage else None
self.inputs = PathSet(inputs)
BaseTarget.__init__(self, self.stampfile, inputs + ([self.depstampfile] if self.depstampfile else []))
class CSharp(BaseTarget):
""" Compile C# files to an executable or dll
"""
compile = None
main = None
libs = None
def __init__(self, output, compile, main=None, libs=None, flags=None, dependencies=None, resources=None):
"""
@param output: the resulting .exe or .dll
@param compile: the input PathSet, path or list of .cs file(s)
@param main: The main class to execute if an exe is to be built.
If this is set then an executable will be created.
Otherwise this target will build a library.
@param libs: a list of input libraries (or a PathSet)
"""
self.compile = FilteredPathSet(_isDotNetFile, PathSet(compile))
self.main = main
self.flags = flags or []
self.libs = PathSet(libs or [])
self.resources = resources or []
BaseTarget.__init__(self, output, [self.compile, self.libs, [x for (x, y) in self.resources], dependencies or []])
self.tags('c#')
def getHashableImplicitInputs(self, context):
return super(CSharp, self).getHashableImplicitInputs(context) + (['main = %s'%context.expandPropertyValues(('%s'%self.main))] if self.main else [])
def run(self, context):
options = context.mergeOptions(self) # get the merged options
libs = self.libs.resolve(context)
libnames = map(lambda x:os.path.basename(x), libs)
libpaths = map(lambda x:os.path.dirname(x), libs)
flags = [context.expandPropertyValues(x) for x in self.flags]
args = [options['csharp.compiler'], "-out:"+self.path]
if libnames: args.extend(["-reference:"+",".join(libnames), "-lib:"+",".join(libpaths)])
if self.main:
args.extend(["-target:exe", "-main:"+self.main])
else:
args.append("-target:library")
for (file, id) in self.resources:
args.append('-resource:%s,%s' % (context.expandPropertyValues(file), context.expandPropertyValues(id)))
args.extend(options['csharp.options'])
args.extend(flags)
args.extend(self.compile.resolve(context))
mkdir(os.path.dirname(self.path))
call(args, outputHandler=options['csharp.processoutputhandler']('csc', False, options=options), timeout=options['process.timeout'])
def __init__(self, target, deps, fn, cleanfn=None): """ @param target: The target file/directory that will be built @param deps: The list of dependencies of this target (paths, pathsets or lists) @param fn: The functor used to build this target @param cleanfn: The functor used to clean this target (optional, defaults to removing the target file/dir) """ BaseTarget.__init__(self, target, deps) self.fn = fn self.cleanfn = cleanfn self.deps = PathSet(deps)
def __init__(self, output, compile, classpath, options=None): """ @param output: output dir for class files @param compile: PathSet (or list) of things to compile @param classpath: PathSet (or list) of things to be on the classpath; @param options: generic target options map for passing options to the underlying operation (optional) """ self.compile = FilteredPathSet(_isJavaFile, PathSet(compile)) self.classpath = PathSet(classpath) BaseTarget.__init__(self, output, [self.compile,self.classpath]) self.options = options
class Javac(BaseTarget):
""" Compile java source to a directory (without jarring it)
"""
compile = None
classpath = None
def __init__(self, output, compile, classpath, options=None):
"""
@param output: output dir for class files
@param compile: PathSet (or list) of things to compile
@param classpath: PathSet (or list) of things to be on the classpath;
@param options: generic target options map for passing options to the
underlying operation (optional)
"""
self.compile = FilteredPathSet(_isJavaFile, PathSet(compile))
self.classpath = PathSet(classpath)
BaseTarget.__init__(self, output, [self.compile,self.classpath])
self.options = options
def run(self, context):
options = context.mergeOptions(self) # get the merged options
# make sure outputdir exists
mkdir(self.path)
# create the classpath, sorting within PathSet (for determinism), but retaining original order of
# PathSet elements in the list
classpath = os.pathsep.join(self.classpath.resolve(context))
# compile everything
mkdir(options.get('javac.logs'))
javac(self.path, self.compile.resolve(context), classpath, options=options, logbasename=options.get('javac.logs')+'/'+targetNameToUniqueId(self.name), targetname=self.name)
def getHashableImplicitInputs(self, context):
# changes in the manifest text should cause a rebuild
# for now, don't bother factoring global jar.manifest.defaults option
# in here (it'll almost never change anyway)
return super(Javac, self).getHashableImplicitInputs(context) + sorted([
'option: %s = "%s"'%(k,v) for (k,v) in context.mergeOptions(self).items()
if v and (k.startswith('javac.') or k == 'java.home')])
def __init__(self, output, compile, main=None, libs=None, flags=None, dependencies=None, resources=None):
"""
@param output: the resulting .exe or .dll
@param compile: the input PathSet, path or list of .cs file(s)
@param main: The main class to execute if an exe is to be built.
If this is set then an executable will be created.
Otherwise this target will build a library.
@param libs: a list of input libraries (or a PathSet)
"""
self.compile = FilteredPathSet(_isDotNetFile, PathSet(compile))
self.main = main
self.flags = flags or []
self.libs = PathSet(libs or [])
self.resources = resources or []
BaseTarget.__init__(self, output, [self.compile, self.libs, [x for (x, y) in self.resources], dependencies or []])
self.tags('c#')
def __init__(self, archivePath, includes=None, excludes=None, destMapper=None): """ @param archivePath: The archive to unpack; either a string or a singleton PathSet @param destMapper: A functor that takes a (context, destPath) where destPath is an archive-relative path (guaranteed to contain / not \\), and returns the desired destination relative path string. The functor should have a deterministic and user-friendly __str__ implementation. @param includes: a list of include patterns (if provided excludes all non-matching files) @param excludes: a list of exclude patterns (processed after includes) """ self.__path = PathSet(archivePath) self.__destMapper = destMapper self.__includes = flatten(includes) self.__excludes = flatten(excludes) self.__location = BuildFileLocation() self.__isResolved = False
def __init__(self, output, jars, keystore, alias=None, storepass=None, manifestDefaults=None): """ @param output: The output directory in which to put the signed jars @param jars: The list (or PathSet) of input jars to copy and sign @param keystore: The path to the keystore @param alias: The alias for the keystore (optional) @param storepass: The password for the store file (optional) @param manifestDefaults: a dictionary of manifest entries to add to the existing manifest.mf file of each jar before signing. Entries in this dictionary will be ignored if the same entry is found in the original manifest.mf file already. """ self.jars = PathSet(jars) self.keystore = keystore self.alias = alias self.storepass = storepass self.manifestDefaults = manifestDefaults BaseTarget.__init__(self, output, [self.jars, self.keystore])
class Custom(BaseTarget): # deprecated because error handling/logging is poor and it promotes bad practices like not using options (e.g process timeout) """ DEPRECATED - use CustomCommand instead, or a dedicated BaseTarget subclass A custom target that builds a single file or directory of content by executing an arbitrary python functor. Functor must take: (target path, [dependency paths], context) Tip: don't forget to ensure the target path's parent dir exists using fileutils.mkdir. @deprecated: Use CustomCommand instead, or a dedicated BaseTarget subclass """ fn = None cleanfn = None def __init__(self, target, deps, fn, cleanfn=None): """ @param target: The target file/directory that will be built @param deps: The list of dependencies of this target (paths, pathsets or lists) @param fn: The functor used to build this target @param cleanfn: The functor used to clean this target (optional, defaults to removing the target file/dir) """ BaseTarget.__init__(self, target, deps) self.fn = fn self.cleanfn = cleanfn self.deps = PathSet(deps) def run(self, context): self.fn(self.path, self.deps.resolve(context), context) def clean(self, context): if self.cleanfn: self.cleanfn(self.path, context) BaseTarget.clean(self, context)
class FilteredArchiveContents(object):
""" Represents an archive to be passed to the Unpack target, with
support for filtering which files are included/excluded, and per-item
destination mapping.
"""
# do NOT use pathset baseclass, because we need the target to handle it in
# a custom way for both per-archived-file mapping and in-archive filtering,
# which would be wrecked by the normalization stuff that pathset does;
# also the model is a bit different for archive contents, so simplest
# just to keep it separate
def __init__(self, archivePath, includes=None, excludes=None, destMapper=None):
"""
@param archivePath: The archive to unpack; either a string or a singleton PathSet
@param destMapper: A functor that takes a (context, destPath) where destPath
is an archive-relative path (guaranteed to contain / not \\), and
returns the desired destination relative path string.
The functor should have a deterministic and
user-friendly __str__ implementation.
@param includes: a list of include patterns (if provided excludes all non-matching files)
@param excludes: a list of exclude patterns (processed after includes)
"""
self.__path = PathSet(archivePath)
self.__destMapper = destMapper
self.__includes = flatten(includes)
self.__excludes = flatten(excludes)
self.__location = BuildFileLocation()
self.__isResolved = False
def getDependency(self):
""" Return the dependency representing this archive (unexpanded and unresolved string, or PathSet).
"""
return self.__path
def getResolvedPath(self, context):
""" Return the fully resolved archive path.
"""
result = self.__path.resolve(context)
if len(result) != 1: raise Exception('Invalid PathSet specified for FilteredArchiveContents, must resolve to exactly one archive: %s'%self.__path)
return result[0]
def isIncluded(self, context, path):
""" Decides whether the specified path within the archive should be
unpacked, based on the include/exclude filters
@param path: a relative path within the archive
"""
if not self.__excludes and not self.__includes: return True
if not self.__isResolved:
self.__includes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__includes])
self.__excludes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__excludes])
self.__isResolved = True
assert '\\' not in path
try:
path = path.lstrip('/')
# first check if it matches an exclude
if next( (True for e in self.__excludes if antGlobMatch(e, path)), False): return False
if not self.__includes: # include everything
return True
else:
m = next( (i for i in self.__includes if antGlobMatch(i, path)), None)
if m:
return True
else:
return False
except Exception, e:
raise BuildException('FilteredArchiveContents error for %s'%(self), causedBy=True, location=self.__location)
class Jar(BaseTarget):
""" Create a Jar, first compiling some java, then packing it all up
"""
compile = None
classpath = None
package = None
manifest = None
def __init__(self,
jar,
compile,
classpath,
manifest,
options=None,
package=None,
preserveManifestFormatting=False):
"""
To add additional entries to the manifest's classpath which are needed at runtime
but not during compilation, use .option('jar.manifest.classpathAppend', [...])
@param jar: path to jar to create
@param compile: PathSet (or list) of things to compile
@param classpath: PathSet (or list) of things to be on the classpath;
destination mapping indicates how they will appear in the manifest
@param manifest: map of manifest entries, OR a string with the filename to use
OR None to disable manifest generation and just produce a normal zip
@param options: [DEPRECATED - use .option() instead]
@param package: PathSet (or list) of other files to include in the jar;
destination mapping indicates where they will appear in the jar
@param preserveManifestFormatting: an advanced option that prevents the jar tool from
reformatting the specified manifest file to comply with Java conventions
(also prevents manifest merging if jar already exists)
"""
self.compile = FilteredPathSet(_isJavaFile,
PathSet(compile)) if compile else None
self.classpath = PathSet(classpath)
self.package = PathSet(package)
self.manifest = manifest
BaseTarget.__init__(self, jar, [
self.compile, self.classpath, self.package,
manifest if isinstance(manifest, basestring) else None
])
for k, v in (options or {}).items():
self.option(k, v)
self.preserveManifestFormatting = preserveManifestFormatting
def run(self, context):
options = self.options
# make sure temp dir exists
mkdir(self.workDir)
classes = os.path.join(self.workDir,
"classes") # output dir for classes
# create the classpath, sorting within PathSet (for determinism), but retaining original order of
# PathSet elements in the list
classpath = os.pathsep.join(self.classpath.resolve(context))
# compile everything
mkdir(
classes
) # (need this for assembling other files to package later on, even if we don't do any javac)
if self.compile:
mkdir(self.getOption('javac.logs'))
javac(classes,
self.compile.resolve(context),
classpath,
options=options,
logbasename=options.get('javac.logs') + '/' +
targetNameToUniqueId(self.name),
targetname=self.name)
manifest = os.path.join(self.workDir, "MANIFEST.MF") # manifest file
if isinstance(self.manifest, basestring):
manifest = context.getFullPath(self.manifest, self.baseDir)
elif self.manifest == None:
manifest = None
else: # generate one
# rewrite property values in the manifest
manifest_entries = {}
for i in self.manifest:
manifest_entries[i] = context.expandPropertyValues(
self.manifest[i])
# determine classpath for manifest
classpath_entries = []
if "Class-path" not in manifest_entries: # assuming it wasn't hardcoded, set it here
for src, dest in self.classpath.resolveWithDestinations(
context):
# we definitely do want to support use of ".." in destinations here, it can be very useful
classpath_entries.append(dest)
assert isinstance(
options['jar.manifest.classpathAppend'], list), options[
'jar.manifest.classpathAppend'] # must not be a string
classpath_entries.extend(
options['jar.manifest.classpathAppend'] or [])
# need to always use / not \ for these to be valid
classpath_entries = [
p.replace(os.path.sep, '/').replace('\\', '/')
for p in classpath_entries if p
]
if classpath_entries:
manifest_entries["Class-path"] = " ".join(
classpath_entries) # include the classpath from here
if not manifest_entries.get(
'Class-path'
): # suppress this element entirely if not needed, otherwise there would be no way to have an empty classpath
manifest_entries.pop('Class-path', '')
# create the manifest file
create_manifest(manifest, manifest_entries, options=options)
# copy in the additional things to include
for (src, dest) in self.package.resolveWithDestinations(context):
if '..' in dest:
raise Exception(
'This target does not permit packaged destination paths to contain ".." relative path expressions'
)
mkdir(os.path.dirname(os.path.join(classes, dest)))
destpath = normLongPath(classes + '/' + dest)
srcpath = normLongPath(src)
if os.path.isdir(srcpath):
mkdir(destpath)
else:
with open(srcpath, 'rb') as s:
with openForWrite(destpath, 'wb') as d:
d.write(s.read())
# create the jar
jar(self.path,
manifest,
classes,
options=options,
preserveManifestFormatting=self.preserveManifestFormatting,
outputHandler=ProcessOutputHandler.create(
'jar', treatStdErrAsErrors=False, options=options))
def getHashableImplicitInputs(self, context):
# changes in the manifest text should cause a rebuild
# for now, don't bother factoring global jar.manifest.defaults option
# in here (it'll almost never change anyway)
return super(Jar, self).getHashableImplicitInputs(context) + [
'manifest = '+context.expandPropertyValues(str(self.manifest)),
'classpath = '+context.expandPropertyValues(str(self.classpath)), # because classpath destinations affect manifest
]+(['preserveManifestFormatting = true'] if self.preserveManifestFormatting else [])\
+sorted(['option: %s = "%s"'%(k,v) for (k,v) in self.options.items()
if v and (k.startswith('javac.') or k.startswith('jar.') or k == 'java.home')])
class Docker(BaseTarget):
""" A target that runs commands in docker, using stamp files for up-to-dateness
"""
BUILD = 1
PUSHTAG = 2
def __init__(self, imagename, mode, inputs, depimage=None, dockerfile=None, buildArgs=None):
"""
imagename: the name/tag of the image to build
"""
self.imagename = imagename
self.depimage = depimage
self.mode = mode
self.dockerfile = dockerfile
self.buildArgs = buildArgs
self.stampfile = '${BUILD_WORK_DIR}/targets/docker/.%s' % re.sub(r'[\\/]', '_', imagename)
self.depstampfile = '${BUILD_WORK_DIR}/targets/docker/.%s' % re.sub(r'[\\/]', '_', depimage) if depimage else None
self.inputs = PathSet(inputs)
BaseTarget.__init__(self, self.stampfile, inputs + ([self.depstampfile] if self.depstampfile else []))
def clean(self, context):
BaseTarget.clean(self, context)
options = context.mergeOptions(self)
args = [ options['docker.path'] ]
environs = { 'DOCKER_HOST' : options['docker.host'] } if options['docker.host'] else {}
args.extend(['rmi', context.expandPropertyValues(self.imagename)])
call(args, outputHandler=options['docker.processoutputhandler']('docker-rmi', False, options=options), timeout=options['process.timeout'], env=environs)
def run(self, context):
options = context.mergeOptions(self)
args = [ options['docker.path'] ]
environs = { 'DOCKER_HOST' : options['docker.host'] } if options['docker.host'] else {}
if self.mode == Docker.BUILD:
dargs = list(args)
dargs.extend([
'build', '--rm=true', '-t', context.expandPropertyValues(self.imagename),
])
if self.buildArgs: dargs.extend(["--build-arg=%s" % [context.expandPropertyValues(x) for x in self.buildArgs]])
if self.dockerfile: dargs.extend(["-f", context.expandPropertyValues(self.dockerfile)])
inputs = self.inputs.resolve(context)
if len(inputs) != 1: raise BuildException("Must specify a single input for Docker.BUILD", location = self.location)
dargs.append(inputs[0])
cwd = os.path.dirname(inputs[0])
call(dargs, outputHandler=options['docker.processoutputhandler']('docker-build', False, options=options), timeout=options['process.timeout'], env=environs, cwd=cwd)
elif self.mode == Docker.PUSHTAG:
inputs = self.inputs.resolve(context)
if len(inputs) != 0: raise BuildException("Must not specify inputs for Docker.PUSHTAG", location = self.location)
dargs = list(args)
dargs.extend([
'tag', context.expandPropertyValues(self.depimage), context.expandPropertyValues(self.imagename),
])
call(dargs, outputHandler=options['docker.processoutputhandler']('docker-tag', False, options=options), timeout=options['process.timeout'], env=environs)
dargs = list(args)
dargs.extend([
'push', context.expandPropertyValues(self.imagename),
])
call(dargs, outputHandler=options['docker.processoutputhandler']('docker-push', False, options=options), timeout=options['process.timeout'], env=environs)
else:
raise BuildException('Unknown Docker mode. Must be Docker.BUILD or Docker.PUSHTAG', location = self.location)
# update the stamp file
path = normLongPath(self.path)
mkdir(os.path.dirname(path))
with openForWrite(path, 'wb') as f:
pass
class CustomCommand(BaseTarget):
"""
A custom target that builds a single file or directory of content by running a
specified command line process.
"""
class __CustomCommandSentinel(object):
def __init__(self, name): self.name = name
def __str__(self): return name
TARGET = __CustomCommandSentinel('TARGET')
DEPENDENCIES = __CustomCommandSentinel('DEPENDENCIES')
def __init__(self, target, command, dependencies, cwd=None, redirectStdOutToTarget=False, env=None, stdout=None, stderr=None):
"""
The command line MUST not reference any generated paths unless they are
explicitly listed in deps.
@param target: the file or directory to be built. Will be cleaned, and its parent dir created,
before target runs.
@param dependencies: an optional list of dependencies; it is essential that ALL dependencies generated by
the build process are explicitly listed here
@param command: a function or a list.
If command is a list, items may be:
- a string (which will be run through expandPropertyValues prior to execution);
must not be used for representing arguments that are paths
- a PathSet (which must resolve to exactly one path - see joinPaths
property functor if multiple paths are required)
- a property functor such as joinPaths (useful for constructing
Java classpaths), basename, etc
- an arbitrary function taking a single context argument
- CustomCommand.TARGET - a special value that is resolved to the
output path of this target
- CustomCommand.DEPENDENCIES - a special value that is resolved to
a list of this target's dependencies
- [deprecated] a ResolvePath(path) object, indicating a path that should be
resolved and resolved at execution time (this is equivalent
to using a PathSet, which is probably a better approach).
If command is a function, must have
signature (resolvedTargetDirPath, resolvedDepsList, context), and
return the command line as a list of strings. resolvedDepsList will be an
ordered, flattened list of resolved paths from deps.
Command lines MUST NOT depend
in any way on the current source or output directory, always use
a PathSet wrapper around such paths.
@param cwd: the working directory to run it from (almost always this should be
left blank, meaning use output dir)
@param env: a dictionary of environment overrides, or a function that
returns one given a context. Values in the dictionary will
be expanded using the same rules as for the command (see above).
Consider using propertyfunctors.joinPaths for environment variables
containing a list of paths.
@param redirectStdOutToTarget: usually, any stdout is treated as logging
and the command is assumed to create the target file itself, but
set this to True for commands where the target file contents are
generated by the stdout of the command being executed.
@param stdout: usually a unique name is auto-generated for .out for this target, but
use this if requried to send output to a specific location.
@param stderr: usually a unique name is auto-generated for .err for this target, but
use this if requried to send output to a specific location.
"""
BaseTarget.__init__(self, target, dependencies)
self.command = command
self.cwd = cwd
self.deps = PathSet(dependencies)
self.redirectStdOutToTarget = redirectStdOutToTarget
if redirectStdOutToTarget and isDirPath(target): raise BuildException('Cannot set redirectStdOutToTarget and specify a directory for the target name - please specify a file instead: %s'%target)
self.env = env
self.stdout, self.stderr = stdout, stderr
if stdout and redirectStdOutToTarget:
raise BuildException('Cannot set both redirectStdOutToTarget and stdout')
def _resolveItem(self, x, context):
if x == self.DEPENDENCIES: return self.deps.resolve(context)
if x == self.TARGET: x = self.path
if isinstance(x, basestring): return context.expandPropertyValues(x)
if hasattr(x, 'resolveToString'): return x.resolveToString(context) # supports Composables too
if isinstance(x, BasePathSet):
result = x.resolve(context)
if len(result) != 1:
raise BuildException('PathSet for custom command must resolve to exactly one path not %d (or use joinPaths): %s'%(len(result), x))
return result[0]
if isinstance(x, ResolvePath): return x.resolve(context, self.baseDir)
raise Exception('Unknown custom command input type %s: %s'%(x.__class__.__name__, x))
def _resolveCommand(self, context):
if callable(self.command):
self.command = self.command(self.path, self.deps.resolve(context), context)
assert not isinstance(self.command, basestring) # must be a list of strings, not a string
self.command = flatten([self._resolveItem(x, context) for x in self.command])
self.command[0] = os.path.normpath(self.command[0])
return self.command
def getHashableImplicitInputs(self, context):
return super(CustomCommand, self).getHashableImplicitInputs(context) + self._resolveCommand(context)
def run(self, context):
if self.cwd: self.cwd = context.getFullPath(self.cwd, self.baseDir)
if isDirPath(self.path):
mkdir(self.path)
cwd = self.cwd or self.path
else:
mkdir(os.path.dirname(self.path))
cwd = self.cwd or self.workDir
mkdir(self.workDir)
cmd = self._resolveCommand(context)
# this location is a lot easier to find than the target's workdir
logbasename = os.path.normpath(context.getPropertyValue('BUILD_WORK_DIR')+'/CustomCommandOutput/'+os.path.basename(cmd[0])+"."+targetNameToUniqueId(self.name))
stdoutPath = context.getFullPath(self.path if self.redirectStdOutToTarget else (self.stdout or logbasename+'.out'), defaultDir='${BUILD_WORK_DIR}/CustomCommandOutput/')
stderrPath = context.getFullPath(self.stderr or logbasename+'.err', defaultDir='${BUILD_WORK_DIR}/CustomCommandOutput/')
self.log.info('Building %s by executing command line: %s', self.name, ''.join(['\n\t"%s"'%x for x in cmd]))
if self.cwd: self.log.info('Building %s from working directory: %s', self.name, self.cwd) # only print if overridden
env = self.env
if env:
if callable(env):
env = env(context)
else:
env = {k: self._resolveItem(env[k], context) for k in env}
self.log.info('Environment overrides for %s are: %s', self.name, ''.join(['\n\t"%s=%s"'%(k, env[k]) for k in env]))
for k in os.environ:
if k not in env: env[k] = os.getenv(k)
self.log.info('Output from %s will be written to "%s" and "%s"', self.name,
stdoutPath,
stderrPath)
if not os.path.exists(cmd[0]) and not (isWindows() and os.path.exists(cmd[0]+'.exe')):
raise BuildException('Cannot run command because the executable does not exist: "%s"'%(cmd[0]), location=self.location)
try:
success=False
rc = None
try:
# maybe send output to a file instead
mkdir(os.path.dirname(logbasename))
with open(stderrPath, 'w') as fe:
with open(stdoutPath, 'w') as fo:
process = subprocess.Popen(cmd,
stderr=fe,
stdout=fo,
cwd=cwd,
env=env)
options = context.mergeOptions(self) # get the merged options
rc = _wait_with_timeout(process, '%s(%s)'%(self.name, os.path.basename(cmd[0])), options['process.timeout'], False)
success = rc == 0
finally:
try:
if os.path.getsize(stderrPath) == 0 and not self.stderr: deleteFile(stderrPath, allowRetry=True)
if not self.redirectStdOutToTarget and os.path.getsize(stdoutPath) == 0 and not self.stdout: deleteFile(stdoutPath, allowRetry=True)
except Exception, e:
# stupid windows, it passes understanding
self.log.info('Failed to delete empty .out/.err files (ignoring error as its not critical): %s', e)
#if not os.listdir(self.workDir): deleteDir(self.workDir) # don't leave empty work dirs around
mainlog = '<command generated no output>'
logMethod = self.log.info if success else self.log.error
if not self.redirectStdOutToTarget and os.path.isfile(stdoutPath) and os.path.getsize(stdoutPath) > 0:
if os.path.getsize(stdoutPath) < 15*1024:
logMethod('Output from %s stdout is: \n%s', self.name, open(stdoutPath, 'r').read().replace('\n', '\n\t'))
mainlog = stdoutPath
if not success: context.publishArtifact('%s stdout'%self, stdoutPath)
if os.path.isfile(stderrPath) and os.path.getsize(stderrPath) > 0:
if os.path.getsize(stderrPath) < 15*1024:
logMethod('Output from %s stderr is: \n%s', self.name, open(stderrPath, 'r').read().replace('\n', '\n\t'))
mainlog = stderrPath # take precedence over stdout
if not success: context.publishArtifact('%s stderr'%self, stderrPath)
if rc != None and rc != 0:
raise BuildException('%s command failed with error code %s; see output at "%s"'%(os.path.basename(cmd[0]), rc, mainlog), location=self.location)
finally:
def __init__(self, target, command, dependencies, cwd=None, redirectStdOutToTarget=False, env=None, stdout=None, stderr=None):
"""
The command line MUST not reference any generated paths unless they are
explicitly listed in deps.
@param target: the file or directory to be built. Will be cleaned, and its parent dir created,
before target runs.
@param dependencies: an optional list of dependencies; it is essential that ALL dependencies generated by
the build process are explicitly listed here
@param command: a function or a list.
If command is a list, items may be:
- a string (which will be run through expandPropertyValues prior to execution);
must not be used for representing arguments that are paths
- a PathSet (which must resolve to exactly one path - see joinPaths
property functor if multiple paths are required)
- a property functor such as joinPaths (useful for constructing
Java classpaths), basename, etc
- an arbitrary function taking a single context argument
- CustomCommand.TARGET - a special value that is resolved to the
output path of this target
- CustomCommand.DEPENDENCIES - a special value that is resolved to
a list of this target's dependencies
- [deprecated] a ResolvePath(path) object, indicating a path that should be
resolved and resolved at execution time (this is equivalent
to using a PathSet, which is probably a better approach).
If command is a function, must have
signature (resolvedTargetDirPath, resolvedDepsList, context), and
return the command line as a list of strings. resolvedDepsList will be an
ordered, flattened list of resolved paths from deps.
Command lines MUST NOT depend
in any way on the current source or output directory, always use
a PathSet wrapper around such paths.
@param cwd: the working directory to run it from (almost always this should be
left blank, meaning use output dir)
@param env: a dictionary of environment overrides, or a function that
returns one given a context. Values in the dictionary will
be expanded using the same rules as for the command (see above).
Consider using propertyfunctors.joinPaths for environment variables
containing a list of paths.
@param redirectStdOutToTarget: usually, any stdout is treated as logging
and the command is assumed to create the target file itself, but
set this to True for commands where the target file contents are
generated by the stdout of the command being executed.
@param stdout: usually a unique name is auto-generated for .out for this target, but
use this if requried to send output to a specific location.
@param stderr: usually a unique name is auto-generated for .err for this target, but
use this if requried to send output to a specific location.
"""
BaseTarget.__init__(self, target, dependencies)
self.command = command
self.cwd = cwd
self.deps = PathSet(dependencies)
self.redirectStdOutToTarget = redirectStdOutToTarget
if redirectStdOutToTarget and isDirPath(target): raise BuildException('Cannot set redirectStdOutToTarget and specify a directory for the target name - please specify a file instead: %s'%target)
self.env = env
self.stdout, self.stderr = stdout, stderr
if stdout and redirectStdOutToTarget:
raise BuildException('Cannot set both redirectStdOutToTarget and stdout')
class Jar(BaseTarget):
""" Create a Jar, first compiling some java, then packing it all up
"""
compile = None
classpath = None
package = None
manifest = None
def __init__(self, jar, compile, classpath, manifest, options=None, package=None, preserveManifestFormatting=False):
"""
@param jar: path to jar to create
@param compile: PathSet (or list) of things to compile
@param classpath: PathSet (or list) of things to be on the classpath;
destination mapping indicates how they will appear in the manifest
@param manifest: map of manifest entries, OR a string with the filename to use
OR None to disable manifest generation and just produce a normal zip
@param options: generic target options map
@param package: PathSet (or list) of other files to include in the jar;
destination mapping indicates where they will appear in the jar
@param preserveManifestFormatting: an advanced option that prevents the jar tool from
reformatting the specified manifest file to comply with Java conventions
(also prevents manifest merging if jar already exists)
"""
self.compile = FilteredPathSet(_isJavaFile, PathSet(compile)) if compile else None
self.classpath = PathSet(classpath)
self.package = PathSet(package)
self.manifest = manifest
BaseTarget.__init__(self, jar, [self.compile,self.classpath,self.package,
manifest if isinstance(manifest, basestring) else None])
self.options = options
self.preserveManifestFormatting = preserveManifestFormatting
def run(self, context):
options = context.mergeOptions(self) # get the merged options
# make sure temp dir exists
mkdir(self.workDir)
classes = os.path.join(self.workDir, "classes") # output dir for classes
# create the classpath, sorting within PathSet (for determinism), but retaining original order of
# PathSet elements in the list
classpath = os.pathsep.join(self.classpath.resolve(context))
# compile everything
mkdir(classes) # (need this for assembling other files to package later on, even if we don't do any javac)
if self.compile:
mkdir(options.get('javac.logs'))
javac(classes, self.compile.resolve(context), classpath, options=options, logbasename=options.get('javac.logs')+'/'+targetNameToUniqueId(self.name), targetname=self.name)
manifest = os.path.join(self.workDir, "MANIFEST.MF") # manifest file
if isinstance(self.manifest, basestring):
manifest = context.getFullPath(self.manifest, self.baseDir)
elif self.manifest == None:
manifest = None
else: # generate one
# rewrite property values in the manifest
manifest_entries = {}
for i in self.manifest:
manifest_entries[i] = context.expandPropertyValues(self.manifest[i])
# determine classpath for manifest
classpath_entries = []
if "Class-path" not in manifest_entries: # assuming it wasn't hardcoded, set it here
for src, dest in self.classpath.resolveWithDestinations(context):
classpath_entries.append(dest)
assert isinstance(options['jar.manifest.classpathAppend'], list), options['jar.manifest.classpathAppend'] # must not be a string
classpath_entries.extend(options['jar.manifest.classpathAppend'] or [])
# need to always use / not \ for these to be valid
classpath_entries = [p.replace(os.path.sep, '/').replace('\\', '/') for p in classpath_entries if p]
if classpath_entries:
manifest_entries["Class-path"] = " ".join(classpath_entries) # include the classpath from here
if not manifest_entries.get('Class-path'): # suppress this element entirely if not needed, otherwise there would be no way to have an empty classpath
manifest_entries.pop('Class-path','')
# create the manifest file
create_manifest(manifest, manifest_entries, options=options)
# copy in the additional things to include
for (src, dest) in self.package.resolveWithDestinations(context):
mkdir(os.path.dirname(os.path.join(classes, dest)))
destpath = normLongPath(classes+'/'+dest)
srcpath = normLongPath(src)
if os.path.isdir(srcpath):
mkdir(destpath)
else:
with open(srcpath, 'rb') as s:
with openForWrite(destpath, 'wb') as d:
d.write(s.read())
# create the jar
jar(self.path, manifest, classes, options=options, preserveManifestFormatting=self.preserveManifestFormatting,
outputHandler=ProcessOutputHandler('jar', treatStdErrAsErrors=False,options=options))
def getHashableImplicitInputs(self, context):
# changes in the manifest text should cause a rebuild
# for now, don't bother factoring global jar.manifest.defaults option
# in here (it'll almost never change anyway)
return super(Jar, self).getHashableImplicitInputs(context) + [
'manifest = '+context.expandPropertyValues(str(self.manifest)),
'classpath = '+context.expandPropertyValues(str(self.classpath)), # because classpath destinations affect manifest
]+(['preserveManifestFormatting = true'] if self.preserveManifestFormatting else [])\
+sorted(['option: %s = "%s"'%(k,v) for (k,v) in context.mergeOptions(self).items()
if v and (k.startswith('javac.') or k.startswith('jar.') or k == 'java.home')])
class SignJars(BaseTarget):
""" Copy jars into a target directory and sign them with the supplied keystore, optionally also updating their manifests
"""
def __init__(self, output, jars, keystore, alias=None, storepass=None, manifestDefaults=None):
"""
@param output: The output directory in which to put the signed jars
@param jars: The list (or PathSet) of input jars to copy and sign
@param keystore: The path to the keystore
@param alias: The alias for the keystore (optional)
@param storepass: The password for the store file (optional)
@param manifestDefaults: a dictionary of manifest entries to add to the existing manifest.mf file
of each jar before signing. Entries in this dictionary will be ignored if the same entry
is found in the original manifest.mf file already.
"""
self.jars = PathSet(jars)
self.keystore = keystore
self.alias = alias
self.storepass = storepass
self.manifestDefaults = manifestDefaults
BaseTarget.__init__(self, output, [self.jars, self.keystore])
def run(self, context):
self.keystore = context.expandPropertyValues(self.keystore)
options = context.mergeOptions(self) # get the merged options
mkdir(self.path)
for src, dest in self.jars.resolveWithDestinations(context):
try:
with open(src, 'rb') as s:
with openForWrite(os.path.join(self.path, dest), 'wb') as d:
d.write(s.read())
shutil.copystat(src, os.path.join(self.path, dest))
# When we re-jar with the user specified manifest entries, jar will complain
# about duplicate attributes IF the original MANIFEST.MF already has those entries.
# This is happening for latest version of SL where Application-Name, Permission etc
# were already there.
#
# The block of code below will first extract the original MANIFEST.MF from the source
# jar file, read all manifest entry to a list. When constructing the new manifest entries,
# make sure the old MANIFEST.MF doesn't have that entry before putting the new manifest entry
# to the list. This will avoid the duplicate attribute error.
#
if self.manifestDefaults:
lines = []
# read each line of MANIFEST.MF of the original jar and put them in lines
with zipfile.ZipFile(src, 'r') as zf:
lst = zf.infolist()
for zi in lst:
fn = zi.filename
if fn.lower().endswith('manifest.mf'):
try:
manifest_txt = zf.read(zi.filename)
except Exception, e:
raise BuildException('Failed reading the manifest file %s with exception:%s' % (fn, e))
# if we have all manifest text, parse and save each line
if manifest_txt:
# CR LF | LF | CR can be there as line feed and hence the code below
lines = manifest_txt.replace('\r\n', '\n').replace('\r','\n').split('\n')
# done
break
original_entries = collections.OrderedDict() # to ensure we don't overwrite/duplicate these
# populate the manifest_entries with original values from original manifest
for l in lines:
if ':' in l and not l.startswith(' '): # ignore continuation lines etc because keys are all we care about
key,value = l.split(':', 1)
original_entries[key] = value.strip()
# build up a list of the new manifest entries (will be merged into any existing manifest by jar)
manifest_entries = collections.OrderedDict()
for i in self.manifestDefaults:
# if entry isn't there yet, add to the list
if i not in original_entries:
manifest_entries[i] = context.expandPropertyValues(self.manifestDefaults[i])
# create the manifest file
# we want to add the manifest entries explicitly specified here but
# NOT the 'default' manifest entries we usually add, since these
# are likely to have been set already, and we do not want duplicates
mkdir(self.workDir)
manifest = os.path.join(self.workDir, "MANIFEST.MF") # manifest file
options['jar.manifest.defaults'] = {}
create_manifest(manifest, manifest_entries, options)
# update the EXISTING jar file with the new manifest entries, which will be merged into
# existing manifest by the jar tool
jar(os.path.join(self.path, dest), manifest, None, options, update=True)
signjar(os.path.join(self.path, dest), self.keystore, options, alias=self.alias, storepass=self.storepass,
outputHandler=ProcessOutputHandler('signjars', treatStdErrAsErrors=False, options=options))
except BuildException, e:
raise BuildException('Error processing %s: %s'%(os.path.basename(dest), e))