def resolveWithDestinations(self, context):
children = flatten([
map(lambda s: s.strip(),
context.expandPropertyValues(c, expandList=True))
for c in self.__children
])
dir = _resolveDirPath(self.__dir, context, self.__location)
result = []
for c in children:
if '*' in c:
raise BuildException(
'Cannot specify "*" patterns here (consider using FindPaths instead): "%s"'
% c,
location=self.__location)
if os.path.isabs(c):
raise BuildException(
'Cannot specify absolute paths here, must be relative (consider using basename): "%s"'
% c,
location=self.__location)
isdir = isDirPath(c)
c = os.path.join(context.expandPropertyValues(dir), c)
c = os.path.normpath(c.rstrip('\\/' + os.path.sep))
if isdir: c = c + os.path.sep
result.append((c, c[len(dir):]))
return result
def run(self, context):
self.log.info("Copying %s to %s", self.src, self.path)
src = self.src.resolveWithDestinations(
context) # a map of srcAbsolute: destRelative
# implicitly ensure parent of target exists, to keep things simple
copied = 0
if not isDirPath(self.name):
# it's a simple file operation.
if len(src) != 1:
raise BuildException(
'Copy destination must be a directory (ending with "/") when multiple sources are specified (not: %s)'
% src)
src, mappedDest = src[0]
if isDirPath(src):
raise BuildException(
'Copy source must be files (or PathSets) not directories: %s'
% src)
mkdir(os.path.dirname(self.path))
self._copyFile(
context, src, self.path
) # we kindof have to ignore mappedDest here, since the target path already fully defines it
if self.mode:
os.chmod(self.path, self.mode)
copied += 1
else:
lastDirCreated = None
for (srcAbs, destRel) in src:
# there should not be any directories here only files from pathsets
if '..' in destRel:
# to avoid people abusing this to copy files outside the dest directory!
raise Exception(
'This target does not permit destination paths to contain ".." relative path expressions'
)
if isDirPath(srcAbs): # allows creating of empty directories.
mkdir(self.path + destRel)
else:
dest = os.path.normpath(self.path + destRel)
#self.log.debug('Processing %s -> %s i.e. %s', srcAbs, destRel, dest)
if not lastDirCreated or lastDirCreated != os.path.dirname(
dest):
lastDirCreated = os.path.dirname(dest)
self.log.debug('Creating intermediate dir %s',
lastDirCreated)
mkdir(lastDirCreated)
try:
self._copyFile(context, srcAbs, dest)
if self.mode:
os.chmod(dest, self.mode)
except Exception, e:
raise BuildException(
'Error copying from "%s" to "%s"' % (srcAbs, dest),
causedBy=True)
copied += 1
def lookupTarget(s):
tfound = init.targets().get(s, None)
if not tfound and '*' in s:
matchregex = s.rstrip('$') + '$'
try:
matchregex = re.compile(matchregex, re.IGNORECASE)
except Exception, e:
raise BuildException(
'Invalid target regular expression "%s": %s' %
(matchregex, e))
matches = [
t for t in init.targets().values()
if matchregex.match(t.name)
]
if len(matches) > 1:
print >> stdout, 'Found multiple targets matching pattern %s:' % (
s)
print >> stdout
for m in matches:
print >> stdout, m.name
print >> stdout
raise BuildException(
'Target regex must uniquely identify a single target: %s (use tags to specify multiple related targets)'
% s)
if matches: return matches[0]
def mergeOptions(self, target=None, options=None):
""" [DEPRECATED] Merges together the default options, the globally overridden options and any set on the target.
DEPRECATED: Use the target's self.options to get the resolved dictionary of options instead of this method.
This is usually called from within a target run() method. Any options provided on the target
will take priority, followed by anything overridden globally, finally anything left is taken
from the option defaults.
@param target: the target from which to take default options. If target is set then the map will
also contain a 'tmpdir' option pointing at the target-specific work directory.
@param options: options to override directly - this is retained for backwards compatibility only and should not be used.
@return: Returns a map of the merged options and their correct values.
"""
# maybe this should move into basetarget eventually
if target:
fulloptions = { 'tmpdir' : target.workDir }
else:
fulloptions = {}
# search defaults, then replace if in globals, then replace if in target.options
for source in [_definedOptions, self._globalOptions, target and target._optionsTargetOverridesUnresolved or {}, options if options else {}]:
if source:
for key in source:
try:
if not key in _definedOptions.keys()+['tmpdir']: raise BuildException("Unknown option %s" % key)
fulloptions[key] = self._recursiveExpandProperties(source[key])
except BuildException, e:
raise BuildException('Failed to resolve option "%s"'%key, location=target.location if target else None, causedBy=True)
def __init__(self, dir, excludes=None, includes=None):
"""
@param dir: base directory to search (relative or absolute, may contain ${...} variables).
May be a simple string, or a DirGeneratedByTarget to glob under a
directory generated as part of the build.
@param includes: a list of glob patterns for the files to include (excluding all others)
@param excludes: a list of glob patterns to exclude after processing any includes.
"""
self.__dir = dir
self.includes = flatten(includes)
self.excludes = flatten(excludes)
bad = [
x for x in (self.includes + self.excludes)
if ('//' in x or x.startswith('/') or x.startswith('/')
or '\\' in x or '${' in x)
]
if bad:
raise BuildException(
'Invalid includes/excludes pattern in FindPaths - must not contain \\, begin or end with /, or contain substitution variables: "%s"'
% bad[0])
if isinstance(
dir, basestring
) and '\\' in dir: # avoid silly mistakes, and enforce consistency
raise BuildException(
'Invalid base directory for FindPaths - must not contain \\ (always use forward slashes)'
)
self.location = BuildFileLocation()
self.__lock = threading.Lock()
self.__cached = None
def __init__(self, init, targets, options):
"""
Create a BuildScheduler.
init - the BuildInitializationContext
targets - the selected targets to build this run (list of target objects)
options - the options for the build (map of string:variable)
"""
self.targetTimes = {} # map of {name : (path, seconds)}
self.targets = None # map of targetPath:BuildTarget where targetPath is the canonical resolved path (from getFullPath)
self.context = None
self.pending = None # list of targetPaths
self.options = None
self.leaves = None
self.lock = None
self.built = 0
self.completed = 0 # include built plus any deemed to be up to date
resetStatCache(
) # at this point reread the stats of files, rather than using potentially stale cached ones
self.targets = {}
caseInsensitivePaths = set()
self.progressFormat = str(len(str(len(init.targets()))))
self.progressFormat = '*** %' + self.progressFormat + 'd/%' + self.progressFormat + 'd '
for t in init.targets().values():
try:
# this is also a good place to resolve target names into paths
t._resolveTargetPath(init)
# do some sanity checking to catch common errors
if t.path.lower() in caseInsensitivePaths:
raise BuildException('Duplicate target path "%s"' %
(t.path))
caseInsensitivePaths.add(t.path.lower())
assert isDirPath(t.path) == isDirPath(t.name), (
repr(t.path), repr(t.name), isDirPath(t.path),
isDirPath(t.name)
) # must have agreement on whether it's a dir or file target between these different representations
for o in init.getOutputDirs():
if t.path.rstrip('\\/') == o.rstrip('\\/'):
raise BuildException(
'Cannot use shared output directory for target: directory targets must always build to a dedicated directory'
)
self.targets[t.path] = BuildTarget(t)
except Exception, e:
if not isinstance(e, IOError):
log.exception(
'FAILED to prepare target %s: ' % t
) # include python stack trace in case it's an xpybuild bug
# ensure all exceptions from here are annotated with the location and target name
raise BuildException('FAILED to prepare target %s' % t,
causedBy=True,
location=t.location)
def enableEnvironmentPropertyOverrides(self, prefix):
if not prefix or not prefix.strip():
raise BuildException('It is mandatory to specify a prefix for enableEnvironmentPropertyOverrides')
# read from env now to save doing it repeatedly later
for k in os.environ:
if k.startswith(prefix):
v = os.environ[k]
k = k[len(prefix):]
if k in self._envPropertyOverrides:
# very unlikely, but useful to check
raise BuildException('Property %s is being read in from the environment in two different ways'%(prefix+k))
self._envPropertyOverrides[k] = v
def defineProperty(self, name, default, coerceToValidValue=None, debug=False):
""" Defines a user-settable property, specifying a default value and
an optional method to validate values specified by the user.
Return the value assigned to the property.
Build files should probably not use this directly, but instead call L{propertysupport.defineStringProperty}
et al. Will raise an exception if called after the build files have been parsed.
@param name: must be UPPER_CASE
@param default: No substitution is performed on this value (see propertysupport.py if you need that).
If set to None, the property must be set on the command line each time
@param coerceToValidValue: None, or a function to validate and/or convert the input string to a value of the right
type
@param debug: if True log at DEBUG else log at INFO
"""
self._initializationCheck()
# enforce naming convention
if name.upper() != name:
raise BuildException('Invalid property name "%s" - all property names must be upper case'%name)
if name in self._properties:
raise BuildException('Cannot set the value of property "%s" more than once'%name)
value = self._propertyOverrides.get(name)
if value==None and name in self._envPropertyOverrides:
value = self._envPropertyOverrides.get(name)
log.critical('Overriding property value from environment: %s=%s', name, value)
if value==None: value = default
if value == None:
raise BuildException('Property "%s" must be set on the command line' % name)
# NB: from this point onwards value may NOT be a string (e.g. could be a boolean)
if coerceToValidValue:
# may change the value (e.g. to make the capitalization consistent or change the type) or throw an
# exception if it's invalid
value = coerceToValidValue(value)
self._properties[name] = value
# remove if still present, so we can tell if user tries to set any undef'd properties
self._propertyOverrides.pop(name, None)
if debug:
log.debug('Setting property %s=%s', name, value)
else:
log.info('Setting property %s=%s', name, value)
return value
def definePropertiesFromFile(propertiesFile,
prefix=None,
excludeLines=None,
conditions=None):
"""
Defines a set of properties from a .properties file
@param propertiesFile: The file to include properties from (can include ${...} variables)
@param prefix: if specified, this prefix will be added to the start of all property names from this file
@param excludeLines: a string of list of strings to search for, any KEY containing these strings will be ignored
@param conditions: an optional list of string conditions that can appear in property
keys e.g. "FOO<condition>=bar" where lines with no condition in this list
are ignored. Conditions are typically lowercase.
"""
if conditions:
assert not isinstance(
conditions, basestring), 'conditions parameter must be a list'
__log.info('Defining properties from file: %s', propertiesFile)
context = getBuildInitializationContext()
propertiesFile = context.getFullPath(
propertiesFile,
BuildFileLocation(raiseOnError=True).buildDir)
try:
f = open(propertiesFile, 'r')
except Exception, e:
raise BuildException('Failed to open properties file "%s"' %
(propertiesFile),
causedBy=True)
def handleEnd(self, returnCode=None):
# linker failures often have no errors but a really useful message in the first warning, so include that in the error message
if returnCode and self.getWarnings() and not self.getErrors():
raise BuildException(
'%s failed with return code %s (first warning: %s)' %
(self._name, returnCode, self.getWarnings()[0]))
ProcessOutputHandler.handleEnd(self, returnCode=returnCode)
def _resolveUnderlyingDependencies(self, context):
r = []
for x in self.contents:
if not isinstance(x, BasePathSet):
r.extend(self.__resolveStringPath(x, context)[0])
else:
# for pathsets, delegate to child
if len(self.contents) == 1:
# short-circuit this common case, avoiding an extra copy and search
return x._resolveUnderlyingDependencies(context)
r.extend(x._resolveUnderlyingDependencies(context))
# check for duplicates here
seen = set()
if len(
self.contents
) > 1: # no point if it's a trivial wrapper around another PathSet
for x in r:
if '//' in x: # this could easily confuse things later!
raise BuildException(
'Invalid path with multiple slashes "%s" in %s' %
(x, self),
location=self.__location)
if x.lower() in seen:
# originally we had an exception here, but occasionally you might have a single thing that's
# a dependency in more than one way (e.g. a jar that's on the classpath AND packaged within an OSGI bundle)
# so just make this a debug log msg
logging.getLogger('PathSet').debug(
'Duplicate PathSet item found: "%s" in %s from %s', x,
self, self.__location)
#raise BuildException('Duplicate item "%s" in %s'%(x, self), location=self.__location)
seen.add(x.lower())
return r
def handleEnd(self, returnCode=None):
if returnCode and not self.getErrors() and fatalerrors:
# special-case to give a more useful error message tha just the exit code if a dependency is missing
raise BuildException(
'Native dependency checking failed: %s' %
(fatalerrors[0]))
return super(VSDependsHandler,
self).handleEnd(returnCode=returnCode)
def __openArchive(self, path):
if path.lower().endswith('.zip') or path.lower().endswith(
'.jar') or path.lower().endswith('.war'):
return zipfile.ZipFile(path, 'r')
if path.lower().endswith('.tar.gz') or path.lower().endswith(
'.tar.bz2') or path.lower().endswith('.tar'):
return tarfile.open(path, 'r')
raise BuildException('Unsupported archive type: %s' % path)
def getTargetsWithTag(self, tag):
""" Returns the list of target objects with the specified tag name
(throws BuildException if not defined).
"""
result = list(self._tags.get(tag, []))
if not result:
raise BuildException('Tag "%s" is not defined for any target in the build'%tag)
return result
def _coerceToValidValue(value):
value = getBuildInitializationContext().expandPropertyValues(
str(value))
if value.lower() == 'true':
return True
if value.lower() == 'false' or value == '':
return False
raise BuildException(
'Invalid property value for "%s" - must be true or false' % (name))
def _defineOption(self, name, default):
""" Register an available option and specify its default value.
Called internally from L{propertysupport.defineOption} and does not
need to be called directly
"""
if name in _definedOptions and _definedOptions[name] != default:
raise BuildException('Cannot define option "%s" more than once'%name)
_definedOptions[name] = default
def processDest(p):
if not p: return p
p = p.replace('\\', '/').split('/')
# p[-1]=='' if p itself is a directory
if len(p) - (0 if p[-1] else 1) > self.__dirsToRemove:
return normpath('/'.join(p[self.__dirsToRemove:]))
else:
raise BuildException(
'Cannot strip %d parent dir(s) from "%s" as it does not have that many parent directories'
% (self.__dirsToRemove, '/'.join(p)))
def __init__(self, name, dependencies):
""" Normal constructor, should only be called from sub-classes since this is a stub.
@param name: a unique name for this target (may contain unexpanded ${...}
variables). Should correspond to the file or directory which is created
as a result of running this target.
@param dependencies: a list of dependencies, which may need to be
flattened/expanded by the build system; may be any combination of
strings, PathSets and lists and may also contain unexpanded variables.
"""
self._optionsTargetOverridesUnresolved = {
} # for target-specific option overrides. for internal use (by buildcontext), do not use
self.__optionsResolved = None # gets assigned during end of initialization phase
if isinstance(name, basestring):
if '//' in name:
raise BuildException(
'Invalid target name: double slashes are not permitted: %s'
% name)
if '\\' in name:
raise BuildException(
'Invalid target name: backslashes are not permitted: %s' %
name)
self.__name = str(name)
self.__path_src = name
self.__tags = ['all']
self.__priority = 0.0 # default so we can go bigger or smaller
self.log = logging.getLogger(self.__class__.__name__)
init = getBuildInitializationContext()
if not init: # doc-test mode
self.location = BuildFileLocation(raiseOnError=False)
else:
self.location = BuildFileLocation(raiseOnError=True)
init.registerTarget(self)
# should ensure changes to the build file cause a rebuild? probs no need
# PathSet will perform all necessary flattening etc
self.__dependencies = PathSet(dependencies)
self.__path = None # set by _resolveTargetPath
self.__workDir = None
def _handle_error(self, target, prefix='Target FAILED'):
""" Perform logging for the exception on the stack, and return an array of
string to be appended to the global build errors list.
target - should be a BaseTarget (not a BuildTarget)
prefix - Prefix of exception, describing what we were doing at the time
"""
e = sys.exc_info()[1]
logged = False
if not isinstance(e, BuildException):
if not isinstance(e, (EnvironmentError)):
# most problems should be wrapped as BuildException already; let's make sure we always
# get an ERROR-level message for things like syntax errors etc
#log.exception('%s: unexpected (non-build) exception in %s'%(prefix, target))
#logged = True
pass # this duplicates the stack trace we get at ERROR level from toMultiLineString
e = BuildException('%s due to %s' % (prefix, e.__class__.__name__),
causedBy=True)
if not logged and log.isEnabledFor(
logging.DEBUG
): # make sure the stack trace is at least available at debug
log.debug('Handling error: %s', traceback.format_exc())
# one-line summary (if in teamcity mode, we'd use teamcity syntax to log this)
#log.error('%s: %s', prefix, e.toSingleLineString(target))
# also useful to have the full stack trace, but only at INFO level
#log.info('%s (details): %s\n', prefix, e.toMultiLineString(target, includeStack=True))
# one-line summary (if in teamcity mode, we'd use teamcity syntax to log this)
#log.error('%s: %s', prefix, e.toSingleLineString(target))
# also useful to have the full stack trace, but only at INFO level
log.error('%s: %s\n',
prefix,
e.toMultiLineString(target, includeStack=True),
extra=e.getLoggerExtraArgDict(target))
return [e.toSingleLineString(target)]
def run(self, context):
self.__unusedMappers = set(self.mappers)
super(FilteredCopy, self).run(context)
if self.__unusedMappers and not self.allowUnusedMappers:
# a useful sanity check, to ensure we're replacing what we think we're replacing, and also that we don't have unused clutter in the build files
raise BuildException(
'Some of the specified mappers did not get used at all during the copy (to avoid confusion, mappers that do not change the output in any way are not permitted): %s'
% (', '.join(
[m.getDescription(context)
for m in self.__unusedMappers])))
class Process(object):
"""
Parent type for compiler/tool chain process classes,
contain helper method for calling the compiler process
"""
def __init__(self, environs=None):
self.environs = environs or {}
def getExpandedEnvirons(self, context, environs=None):
"""
Return the environment variables to be used for this process,
either passed explicitly into this method, or passed into
the constructor, or else empty.
Any ${...} substitution vars are expanded automatically.
"""
environs = environs or self.environs or {}
environs = environs.copy()
for k in environs:
environs[k] = context.expandPropertyValues(environs[k])
return environs
def call(self,
context,
args,
outputHandler,
options,
cwd=None,
environs=None):
try:
args = flatten([
context.expandPropertyValues(x, expandList=True) for x in args
])
try:
outputHandlerInstance = outputHandler(os.path.basename(
args[0]),
options=options)
except Exception, e:
# backwards compatibility for output handlers that don't pass kwargs down
outputHandlerInstance = outputHandler(os.path.basename(
args[0]))
call(args,
outputHandler=outputHandlerInstance,
cwd=cwd,
env=self.getExpandedEnvirons(context, environs),
timeout=options['process.timeout'])
except BuildException as e:
# causedBy is not useful here
raise BuildException("%s process failed" %
(os.path.basename(args[0])),
causedBy=True)
def run(self, context):
mkdir(os.path.dirname(self.path))
alreadyDone = set()
with zipfile.ZipFile(normLongPath(self.path), 'w') as output:
for (f, o) in self.inputs.resolveWithDestinations(context):
# if we don't check for duplicate entries we'll end up creating an invalid zip
if o in alreadyDone:
dupsrc = ['"%s"'%src for (src, dest) in self.inputs.resolveWithDestinations(context) if dest == o]
raise BuildException('Duplicate zip entry "%s" from: %s'%(o, ', '.join(dupsrc)))
alreadyDone.add(o)
# can't compress directory entries! (it messes up Java)
output.write(normLongPath(f).rstrip('/\\'), o, zipfile.ZIP_STORED if isDirPath(f) else zipfile.ZIP_DEFLATED)
def _coerceToValidValue(value):
value = getBuildInitializationContext().expandPropertyValues(value)
if value in enumValues: return value
# case-insensitive match
for e in enumValues:
if e.lower() == value.lower():
return e
raise BuildException(
'Invalid property value for "%s" - value "%s" is not one of the allowed enumeration values: %s'
% (name, value, enumValues))
def getPropertyValue(self, name):
""" Get the value of the specified property or raise a BuildException if it doesn't exist.
@param name: the property name (without ${...}) to retrieve. Must be a string.
@return: For Boolean properties this will be a python Boolean, for everything else it will be a string.
>>> BaseContext({'A':'b','BUILD_MODE':'release'}).getPropertyValue('BUILD_MODE')
'release'
>>> BaseContext({'A':False}).getPropertyValue('A')
False
>>> BaseContext({'A':True}).getPropertyValue('A')
True
>>> BaseContext({'A':'b'}).getPropertyValue('UNDEFINED_PROPERTY')
Traceback (most recent call last):
...
BuildException: Property "UNDEFINED_PROPERTY" is not defined
>>> BaseContext({'A':'b'}).getPropertyValue(None)
Traceback (most recent call last):
...
BuildException: Property "None" is not defined
>>> BaseContext({'A':'b'}).getPropertyValue('')
Traceback (most recent call last):
...
BuildException: Property "" is not defined
"""
result = self._properties.get(name)
if result == None:
# before throwing an exception, see if it's one of the special
# always-defined proprties. During parse phase we don't auto-define
# these until they are needed in case build wants to override them
# nb: each item here corresponds to a getPropertyValue() call in initializeFromBuildFile
if name=='OUTPUT_DIR':
outputDir = getBuildInitializationContext().defineProperty('OUTPUT_DIR', getBuildInitializationContext()._rootDir+'/'+'buildoutput')
getBuildInitializationContext().registerOutputDir(outputDir)
elif name=='BUILD_MODE':
getBuildInitializationContext().defineProperty('BUILD_MODE', 'release')
elif name=='BUILD_NUMBER':
getBuildInitializationContext().defineProperty('BUILD_NUMBER', str(int(time.time()))) # this is a suitably unique number (perhaps too unique? don't want to force rebuilds...)
elif name=='BUILD_WORK_DIR':
getBuildInitializationContext().defineProperty('BUILD_WORK_DIR',
os.path.normpath(getBuildInitializationContext().getPropertyValue('OUTPUT_DIR')+'/BUILD_WORK'))
elif name=='LOG_FILE':
getBuildInitializationContext().defineProperty('LOG_FILE', os.path.abspath("build.log"))
else:
raise BuildException('Property "%s" is not defined'%name)
result = self._properties.get(name)
assert result!=None, name
return result
def _resolveDirPath(dir, context, location):
"""
Helper method for use within this module:
resolves a single directory path that may be either a string or a DirGeneratedByTarget.
This path is guaranteed to be expanded and to end with a trailing / character.
"""
if isinstance(dir, BasePathSet):
dir = dir.resolve(context)
if len(dir) != 1:
raise BuildException(
'This PathSet requires exactly one base directory but %d were provided: %s'
% (len(dir), dir),
location=location)
dir = dir[0]
else:
dir = context.getFullPath(dir, defaultDir=location)
if not isDirPath(dir):
raise BuildException(
'Directory paths must end with an explicit / slash: "%s"' % dir,
location=location)
return dir
def setGlobalOption(self, key, value):
""" Set a global value for an option
Called internally from L{propertysupport.setGlobalOption} and does not
need to be called directly
"""
if not key in _definedOptions:
raise BuildException("Cannot specify value for option that has not been defined \"%s\"" % key)
if key in self._globalOptions:
log.warn("Resetting global option %s to %s at %s", key, value, BuildFileLocation().getLineString())
else:
log.info("Setting global option %s to %s at %s", key, value, BuildFileLocation().getLineString())
self._globalOptions[key] = value
def __init__(self, dirTargetName):
"""
@param dirTargetName: The directory that another target will generate.
"""
BasePathSet.__init__(self)
assert isinstance(dirTargetName, basestring)
assert dirTargetName.endswith('/')
self.__target = dirTargetName
self.__location = BuildFileLocation()
if '\\' in dirTargetName: # avoid silly mistakes, and enforce consistency
raise BuildException(
'Invalid directory target - must not contain \\ (always use forward slashes)'
)
def _resolveUnderlyingDependencies(self, context):
# sanity check to avoid user error (but not in doctest mode)
if hasattr(context, '_isValidTarget') and not context._isValidTarget(
self.__target):
raise BuildException(
'Unknown target name specified for DirGeneratedByTarget: "%s"'
% self.__target,
location=self.__location)
# don't need to do anything complicated here,
# the magic is that all parent pathsets this is wrapped inside
# will always delegate down to this path
return self.resolve(context)
def registerTarget(self, target):
""" Registers the target with the context.
Called internally from L{basetarget.BaseTarget} and does not need to be called directly.
Will raise an exception if called after the build files have been parsed.
"""
self._initializationCheck()
# don't do anything much with it yet, wait for initialization phase to complete first
if target.name in self._targetsMap:
raise BuildException('Duplicate target name "%s" (%s)' % (target, self._targetsMap[target.name].location), location=target.location)
self._targetsMap[target.name] = target
self._targetsList.append(target)
self.registerTags(target, target.getTags())
def _resolveUnderlyingDependencies(self, context):
deplist = None
options = self.target.options # get the merged options
dfile = normLongPath(self.target.workDir+'.makedepend')
testsources = self.sources.resolve(context)
depsources = self.sources._resolveUnderlyingDependencies(context)
needsRebuild = not os.path.exists(dfile)
if needsRebuild:
self.log.info("Rebuilding dependencies for %s because cached dependencies file does not exist (%s)" % (self.target, dfile))
dfiletime = 0 if needsRebuild else getmtime(dfile)
for x in testsources:
if not exists(x):
# can't generate any deps if some source files don't yet exist
self.log.info("Dependency generation %s postponed because source file does not exist: %s" % (self.target, x))
return depsources
elif getmtime(x) > dfiletime:
if not needsRebuild: self.log.info("Rebuilding dependencies for %s because cached dependencies file is older than %s" % (self.target, x))
needsRebuild = True
if not needsRebuild: # read in cached dependencies
deplist = []
with open(dfile) as f:
lines = f.readlines()
header = lines[0].strip()
lines = lines[1:]
for d in lines:
d = d.strip()
if context._isValidTarget(d) or exists(normLongPath(d)):
deplist.append(d)
else:
needsRebuild = True
self.log.warn("Rebuilding dependencies for %s because dependency %s is missing" % (self.target, d))
break
if header != str(self):
self.log.info("Rebuilding dependencies for %s because target options have changed (%s != %s)" % (self.target, header, str(self)))
elif not needsRebuild:
return deplist
# generate them again
startt = time.time()
self.log.info("*** Generating native dependencies for %s" % self.target)
try:
deplist = options['native.compilers'].dependencies.depends(context=context, src=testsources, options=options, flags=flatten(options['native.cxx.flags']+[context.expandPropertyValues(x).split(' ') for x in self.flags]), includes=flatten(self.includes.resolve(context)+[context.expandPropertyValues(x, expandList=True) for x in options['native.include']]))
except BuildException, e:
if len(testsources)==1 and testsources[0] not in str(e):
raise BuildException('Dependency resolution failed for %s: %s'%(testsources[0], e))
raise
def _handle_error(self, target, prefix='Target FAILED'):
""" Perform logging for the exception on the stack, and return an array of
string to be appended to the global build errors list.
target - should be a BaseTarget (not a BuildTarget)
prefix - Prefix of exception, describing what we were doing at the time
"""
e = sys.exc_info()[1]
logged = False
if not isinstance(e, BuildException):
if not isinstance(e, (EnvironmentError)):
# most problems should be wrapped as BuildException already; let's make sure we always
# get an ERROR-level message for things like syntax errors etc
#log.exception('%s: unexpected (non-build) exception in %s'%(prefix, target))
#logged = True
pass # this duplicates the stack trace we get at ERROR level from toMultiLineString
e = BuildException('%s due to %s'%(prefix, e.__class__.__name__), causedBy=True)
if not logged and log.isEnabledFor(logging.DEBUG): # make sure the stack trace is at least available at debug
log.debug('Handling error: %s', traceback.format_exc())
# one-line summary (if in teamcity mode, we'd use teamcity syntax to log this)
#log.error('%s: %s', prefix, e.toSingleLineString(target))
# also useful to have the full stack trace, but only at INFO level
#log.info('%s (details): %s\n', prefix, e.toMultiLineString(target, includeStack=True))
# one-line summary (if in teamcity mode, we'd use teamcity syntax to log this)
#log.error('%s: %s', prefix, e.toSingleLineString(target))
# also useful to have the full stack trace, but only at INFO level
log.error('%s: %s\n', prefix, e.toMultiLineString(target, includeStack=True), extra=e.getLoggerExtraArgDict(target))
return [e.toSingleLineString(target)]
def _deps_target(self, tname):
"""
Function called by a worker to check the deps for a single target
tname - this is the canonical PATH of the target, not the name
"""
errors = []
pending = [] # list of new jobs to done as part of dependency resolution
log.debug("Inspecting dependencies of target %s", tname)
target = self.targets.get(tname, None)
# only log dependency status periodically since usually its very quick
# and not worthwhile
with self.lock:
self.index += 1
log.critical(self.progressFormat+"Resolving dependencies for %s", self.index, self.total, target)
if not target:
assert False # I'm not sure how we can get here, think it should actually be impossible
if not exists(tname):
errors.append("Unknown target %s" % tname)
else:
log.debug('Scheduler cannot find target in build file or on disk: %s', target) # is this a problem? maybe assert False here?
elif self.options['ignore-deps'] and exists(target.path):
# in this mode, any target that already exists should be treated as
# a leaf with no deps which means it won't be built under any
# circumstances (even if a target it depends on is rebuilt),
# and allows us to avoid the time-consuming transitive resolution
# of dependencies. Has to be implemented this way, since if we were
# to allow ANY already-existing target to be re-built in the normal
# way, we would have to resolve dependencies for all targets in
# order to ensure we never rebuild a target at the same time as
# a target that depends on it. We're essentially deleting the entire
# dependency subtree for all nodes that exist already
log.debug('Scheduler is treating existing target as a leaf and will not rebuild it: %s', target)
self.leaves.append(target)
elif not (self.options['ignore-deps'] and self.options['clean']):
try:
deps = target.resolveDependencies(self.context)
if deps: log.debug('%s has %d dependencies', target.target, len(deps))
targetDeps = [] # just for logging
leaf = True
for dname in deps:
#log.debug('Processing dependency: %s -> %s', tname, dname)
dpath = normLongPath(dname)
if dname in self.targets:
leaf = False
dtarget = self.targets[dname]
if dtarget in target.rdeps(): raise Exception('Circular dependency between targets: %s and %s'%(dtarget.name, target.name))
dtarget.rdep(target)
self._updatePriority(target)
target.increment()
if not isDirPath(dname):
target.filedep(dname) # might have an already built target dependency which is still newer
else:
# special case directory target deps - must use stamp file not dir, to avoid re-walking
# the directory needlessly, and possibly making a wrong decision if the dir pathset is
# from a filtered pathset
target.filedep(self.targets[dname].stampfile)
with self.lock:
if not dname in self.pending:
self.pending.append(dname)
pending.append((0, dname))
targetDeps.append(str(self.targets[dname]))
elif (isDirPath(dname) and isdir(dpath)) or (not isDirPath(dname) and isfile(dpath)):
target.filedep(dname)
else:
# in the specific case of a dependency error, build will definitely fail immediately so we should log line number
# at ERROR log level not just at info
ex = BuildException("Cannot find dependency %s" % dname)
log.error('FAILED during dependency resolution: %s', ex.toMultiLineString(target, includeStack=False), extra=ex.getLoggerExtraArgDict(target))
assert not os.path.exists(dpath), dname
errors.append(ex.toSingleLineString(target))
break
if leaf:
log.info('Target dependencies of %s (priority %s) are: <no dependencies>', target, -target.priority)
self.leaves.append(target)
else:
log.info('Target dependencies of %s (priority %s) are: %s', target, -target.priority, ', '.join(targetDeps)) # this is important for debugging missing dependencies etc
except Exception as e:
errors.extend(self._handle_error(target.target, prefix="Target FAILED during dependency resolution"))
else:
# For clean ignoring deps we want to be as light-weight as possible
self.leaves.append(target)
if pending:
# if we're adding some new jobs
with self.lock:
self.total += len(pending)
# NB: the keep-going option does NOT apply to dependency failures
return (pending, errors, 0 == len(errors))
