def run(self, context):
self.log.info('Cleaning existing files from %s', self.path)
deleteDir(self.path)
iswindows = IS_WINDOWS
for a in self.archives:
a_is_filteredarchivecontents = isinstance(a, FilteredArchiveContents)
if a_is_filteredarchivecontents:
items = [(a.getResolvedPath(context), '')]
else:
assert isinstance(a, BasePathSet)
filteredMembers = None
items = a.resolveWithDestinations(context)
for (srcAbs, destRel) in items:
if destRel and not isDirPath(destRel): destRel = os.path.dirname(destRel) # strip off the zip filename
if '..' in destRel: raise Exception('This target does not permit destination paths to contain ".." relative path expressions')
try:
filesize = os.path.getsize(srcAbs)
except Exception:
filesize = 0
self.log.info("Unpacking %s (%0.1f MB) to %s", os.path.basename(srcAbs), filesize/1024.0/1024, self.name+destRel)
starttime = time.time()
with self. __openArchive(srcAbs) as f:
mkdir(self.path+destRel)
if a_is_filteredarchivecontents and a.hasIncludeExcludeFilters():
fullList = _getnames(f)
if not fullList:
raise BuildException('No files were found in archive "%s"'%(srcAbs))
filteredMembers = [x for x in fullList if a.isIncluded(context, x)]
self.log.info("Unpacking %d of %d members in %s", len(filteredMembers), len(fullList), os.path.basename(srcAbs))
if not filteredMembers:
raise BuildException('No files matching the specified include/exclude filters were found in archive "%s": %s'%(srcAbs, a))
if len(filteredMembers)==len(fullList):
raise BuildException('No files were excluded from the unpacking operation by the specified filters (check filters are correct): %s'%a)
else:
filteredMembers = _getnames(f)
# NB: some archive types want a list of string members, others want TarInfo objects etc, so
# if we support other archive types in future might need to do a bit of work here
path = normLongPath(self.path+destRel)
for m in filteredMembers:
if not isDirPath(m):
info = _getinfo(f, m)
if a_is_filteredarchivecontents:
_setfilename(info, a.mapDestPath(context, _getfilename(info)))
if iswindows: _setfilename(info, _getfilename(info).replace('/', '\\'))
f.extract(info, path=path)
else:
# we should create empty directories too
if a_is_filteredarchivecontents:
m = a.mapDestPath(context, m).rstrip('/')
m = path.rstrip('/\\')+'/'+m
if iswindows: m = m.replace('/', '\\')
mkdir(m)
self.log.info("Completed unpacking %s (%0.1f MB) in %0.1f seconds", os.path.basename(srcAbs), filesize/1024.0/1024, (time.time()-starttime))
def __init__(self,
target,
command=None,
dependencies=[],
cwd=None,
redirectStdOutToTarget=False,
env=None,
stdout=None,
stderr=None,
commands=None):
BaseTarget.__init__(self, target, dependencies)
assert not (command
and commands), 'Cannot specify both command= and commands='
self.command = command
self.commands = commands
self.cwd = cwd
self.deps = PathSet(dependencies)
self.redirectStdOutToTarget = redirectStdOutToTarget
if redirectStdOutToTarget and isDirPath(target):
raise BuildException(
'Cannot set redirectStdOutToTarget and specify a directory for the target name - please specify a file instead: %s'
% target)
self.env = env
self.stdout, self.stderr = stdout, stderr
if stdout and redirectStdOutToTarget:
raise BuildException(
'Cannot set both redirectStdOutToTarget and stdout')
def resolveToString(self, context):
"""
.. private:: There is usually no need for this to be called other than by the framework.
Resolves this target's path and returns as a string.
It is acceptable to call this while the build files are still being
parsed (before the dependency checking phase), but an error will result
if resolution depends on anything that has not yet been defined.
"""
# implementing this allows targets to be used in Composeable expressions
# if there's no explicit parent, default to ${OUTPUT_DIR} to stop
# people accidentally writing to their source directories
if self.__path is not None:
return self.__path # cache it for consistency
self.__path = context.getFullPath(
self.__path_src, context.getPropertyValue("OUTPUT_DIR"))
badchars = '<>:"|?*' # Windows bad characters; it's helpful to stop people using such characters on all OSes too since almost certainly not intended
foundbadchars = [
c for c in self.__path[2:] if c in badchars
] # (nb: ignore first 2 chars of absolute path which will necessarily contain a colon on Windows)
if foundbadchars:
raise BuildException(
'Invalid character(s) "%s" found in target name %s' %
(''.join(sorted(list(set(foundbadchars)))), self.__path))
if self.__path.endswith(('.', ' ')):
raise BuildException(
'Target name must not end in a "." or " "'
) # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
self.log.debug('Resolved target name %s to canonical path %s',
self.name, self.path)
return self.__path
def __init__(self, dest, archives):
"""
@param dest: the output directory (ending with a "/"). Never
specify a dest directory that is also written to by another
target (e.g. do not specify a build 'output' directory here).
@param archives: the input archives to be unpacked, which may be any
combination of strings, PathSets, FilteredArchiveContents and lists of these.
If these PathSets include mapping information, this
will be used to define where (under the dest directory) each
file from within that archive is copied (but cannot be used to
change the archive-relative path of each item).
For advanced cases, FilteredArchiveContents can be used to provide
customized mapping and filtering of the archive contents,
including manipulation of the destinations encoded within the
archive itself.
"""
if not dest.endswith('/'): raise BuildException('Unpack target destination must be a directory (ending with "/"), not: "%s"'%dest)
# NB: we could also support copying in non-archived files into the directory future too
# we should preserve the specified order of archives since it may
# affect what happens when they contain the same files and must
# overwrite each other
archives = [a if (isinstance(a, BasePathSet) or isinstance(a, FilteredArchiveContents)) else PathSet(a) for a in flatten(archives)]
BaseTarget.__init__(self, dest, [
(a.getDependency() if isinstance(a, FilteredArchiveContents) else a)
for a in archives])
self.archives = archives
def isIncluded(self, context, path):
""" Decides whether the specified path within the archive should be
unpacked, based on the include/exclude filters
@param path: a relative path within the archive
"""
if not self.__excludes and not self.__includes: return True
if not self.__isResolved:
self.__includes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__includes])
self.__excludes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__excludes])
self.__isResolved = True
assert '\\' not in path
try:
path = path.lstrip('/')
# first check if it matches an exclude
if next( (True for e in self.__excludes if antGlobMatch(e, path)), False): return False
if not self.__includes: # include everything
return True
else:
m = next( (i for i in self.__includes if antGlobMatch(i, path)), None)
if m:
return True
else:
return False
except Exception as e:
raise BuildException('FilteredArchiveContents error for %s'%(self), causedBy=True, location=self.__location)
def __init__(self, extToEncoding={}, default=None):
self.extToEncodingDict, self.defaultEncoding = dict(extToEncoding), default
# enforce starts with a . to prevent mistakes and allow us to potentially optimize the implementation in future
for k in extToEncoding:
if not k.startswith('.'): raise BuildException(f'ExtensionBasedFileEncodingDecider extension does not start with the required "." character: "{k}"')
self.__cache = None,None
self.__stringified = f'ExtensionBasedFileEncodingDecider({self.extToEncodingDict}; default={self.defaultEncoding})'
def setGlobalOption(self, key, value):
"""
.. private:: For internal use only.
Set a global value for an option
Called internally from L{propertysupport.setGlobalOption} and does not
need to be called directly
"""
if not key in BuildInitializationContext._definedOptions:
raise BuildException(
"Cannot specify value for option that has not been defined \"%s\""
% key)
if key in self._globalOptions:
# for a like-to-like comparison, need to expand any properties (empirically one of them seems to be non-expanded)
if self._recursiveExpandProperties(
value) != self._recursiveExpandProperties(
self._globalOptions[key]):
log.warn(
"Resetting global option %s to %s at %s; old value was %s",
key, self._recursiveExpandProperties(value),
BuildFileLocation().getLineString(),
self._recursiveExpandProperties(self._globalOptions[key]))
else:
log.info("Setting global option %s to %s at %s", key, value,
BuildFileLocation().getLineString())
self._globalOptions[key] = value
def _coerceToValidValue(value):
value = BuildInitializationContext.getBuildInitializationContext().expandPropertyValues(str(value))
if value.lower() == 'true':
return True
if value.lower() == 'false' or value=='':
return False
raise BuildException('Invalid property value for "%s" - must be true or false' % (name))
def handleEnd(self, returnCode=None):
# linker failures often have no errors but a really useful message in the first warning, so include that in the error message
if returnCode and self.getWarnings() and not self.getErrors():
raise BuildException(
'%s failed with return code %s (first warning: %s)' %
(self._name, returnCode, self.getWarnings()[0]))
ProcessOutputHandler.handleEnd(self, returnCode=returnCode)
def handleEnd(self, returnCode=None):
"""
Called when the process has terminated, to collate any warnings, errors and other messages,
and in conjunction with the ``returnCode`` optionally raise an `xpybuild.utils.buildexceptions.BuildException`
with a suitable message.
The default implementation logs a message summarizing the total number of warnings, then raises a
``BuildException`` if there were any error or (unless ``ignoreReturnCode`` was set) if ``returnCode`` is
non-zero. The exception message will contain the first error, or if none the first warning, or failing that,
the last line of the output.
"""
if self._warnings: self._logger.warning('%d warnings during %s', len(self._warnings), self._name)
if self._errors:
msg = self._errors[0]
if len(self._errors)>1:
msg = '%d errors, first is: %s'%(len(self._errors), msg)
elif returnCode and not self._ignoreReturnCode:
msg = '%s failed with return code %s'%(self._name, returnCode)
# in case it's relevant, since the return code doesn't provide much to go on
if self._warnings:
msg += '; no errors reported, first warning was: %s'%self._warnings[0]
elif self.getLastOutputLine():
# last-ditch attempt to be useful
msg += '; no errors reported, last line was: %s'%self.getLastOutputLine()
else:
msg += ' and no output generated'
else:
return
raise BuildException(msg)
def call(self,
context,
args,
outputHandler,
options,
cwd=None,
environs=None):
try:
args = flatten([
context.expandPropertyValues(x, expandList=True) for x in args
])
try:
outputHandlerInstance = outputHandler(os.path.basename(
args[0]),
options=options)
except Exception as e:
# backwards compatibility for output handlers that don't pass kwargs down
outputHandlerInstance = outputHandler(os.path.basename(
args[0]))
call(args,
outputHandler=outputHandlerInstance,
cwd=cwd,
env=self.getExpandedEnvirons(context, environs),
timeout=options['process.timeout'])
except BuildException as e:
# causedBy is not useful here
raise BuildException("%s process failed" %
(os.path.basename(args[0])),
causedBy=True)
def handleEnd(self, returnCode=None):
if returnCode and not self.getErrors() and fatalerrors:
# special-case to give a more useful error message tha just the exit code if a dependency is missing
raise BuildException(
'Native dependency checking failed: %s' %
(fatalerrors[0]))
return super(VSDependsHandler,
self).handleEnd(returnCode=returnCode)
def enableEnvironmentPropertyOverrides(self, prefix):
if not prefix or not prefix.strip():
raise BuildException(
'It is mandatory to specify a prefix for enableEnvironmentPropertyOverrides'
)
# read from env now to save doing it repeatedly later
for k in os.environ:
if k.startswith(prefix):
v = os.environ[k]
k = k[len(prefix):]
if k in self._envPropertyOverrides:
# very unlikely, but useful to check
raise BuildException(
'Property %s is being read in from the environment in two different ways'
% (prefix + k))
self._envPropertyOverrides[k] = v
def getTargetsWithTag(self, tag):
""" Returns the list of target objects with the specified tag name
(throws BuildException if not defined).
"""
result = list(self._tags.get(tag, []))
if not result:
raise BuildException(
'Tag "%s" is not defined for any target in the build' % tag)
return result
def _coerceToValidValue(value):
value = BuildInitializationContext.getBuildInitializationContext().expandPropertyValues(value)
if value in enumValues: return value
# case-insensitive match
for e in enumValues:
if e.lower()==value.lower():
return e
raise BuildException('Invalid property value for "%s" - value "%s" is not one of the allowed enumeration values: %s' % (name, value, enumValues))
def _handle_error(self, target, prefix='Target FAILED'):
""" Perform logging for the exception on the stack, and return an array of
string to be appended to the global build errors list.
target - should be a BaseTarget (not a TargetWrapper)
prefix - Prefix of exception, describing what we were doing at the time
"""
e = sys.exc_info()[1]
logged = False
if not isinstance(e, BuildException):
if not isinstance(e, (EnvironmentError)):
# most problems should be wrapped as BuildException already; let's make sure we always
# get an ERROR-level message for things like syntax errors etc
#log.exception('%s: unexpected (non-build) exception in %s'%(prefix, target))
#logged = True
pass # this duplicates the stack trace we get at ERROR level from toMultiLineString
e = BuildException('%s due to %s' % (prefix, e.__class__.__name__),
causedBy=True)
if not logged and log.isEnabledFor(
logging.DEBUG
): # make sure the stack trace is at least available at debug
log.debug('Handling error: %s', traceback.format_exc())
# one-line summary (if in teamcity mode, we'd use teamcity syntax to log this)
#log.error('%s: %s', prefix, e.toSingleLineString(target))
# also useful to have the full stack trace, but only at INFO level
#log.info('%s (details): %s\n', prefix, e.toMultiLineString(target, includeStack=True))
# one-line summary (if in teamcity mode, we'd use teamcity syntax to log this)
#log.error('%s: %s', prefix, e.toSingleLineString(target))
# also useful to have the full stack trace, but only at INFO level
log.error('%s: %s\n',
prefix,
e.toMultiLineString(target, includeStack=True),
extra=e.getLoggerExtraArgDict(target))
return [e.toSingleLineString(target)]
def run(self, context):
self.__unusedMappers = set(self.mappers)
super(FilteredCopy, self).run(context)
if self.__unusedMappers and not self.allowUnusedMappers:
# a useful sanity check, to ensure we're replacing what we think we're replacing, and also that we don't have unused clutter in the build files
raise BuildException(
'Some of the specified mappers did not get used at all during the copy (to avoid confusion, mappers that do not change the output in any way are not permitted): %s'
% (', '.join(
[m.getDescription(context)
for m in self.__unusedMappers])))
def _mergeListOfOptionDicts(self, dicts, target=None):
# creates a new dictionary
# dicts is a list of dictionaries
# target is used just for error reporting, if available
fulloptions = {}
for source in dicts:
if source is None: continue
for key in source:
try:
if key not in BuildInitializationContext._definedOptions:
raise BuildException("Unknown option %s" % key)
fulloptions[key] = self._recursiveExpandProperties(
source[key])
except BuildException as e:
raise BuildException(
'Failed to resolve option "%s"' % key,
location=target.location if target else None,
causedBy=True) from None
return fulloptions
def checkForNonTargetDependenciesUnderOutputDirs(self):
"""
Iterate over the dependencies that are not targets and return
the name of one that's under an output dir (suggests a missing target
dep), or None.
"""
ctx = self.__scheduler.context
for dpath, flags, pathset in self.__nontargetdeps:
if ctx.isPathWithinOutputDir(dpath):
raise BuildException('Target %s depends on output %s which is implicitly created by some other directory target - please use DirGeneratedByTarget to explicitly name the directory target that it depends on'%(self, dpath),
location=self.target.location) # e.g. FindPaths(DirGeneratedByTarget('${OUTPUT_DIR}/foo/')+'bar/') # or similar
def run(self, context):
mkdir(os.path.dirname(self.path))
alreadyDone = set()
with zipfile.ZipFile(normLongPath(self.path), 'w') as output:
for (f, o) in self.inputs.resolveWithDestinations(context):
# if we don't check for duplicate entries we'll end up creating an invalid zip
if o in alreadyDone:
dupsrc = ['"%s"'%src for (src, dest) in self.inputs.resolveWithDestinations(context) if dest == o]
raise BuildException('Duplicate zip entry "%s" from: %s'%(o, ', '.join(dupsrc)))
alreadyDone.add(o)
# can't compress directory entries! (it messes up Java)
output.write(normLongPath(f).rstrip('/\\'), o, zipfile.ZIP_STORED if isDirPath(f) else zipfile.ZIP_DEFLATED)
def lookupTarget(s):
tfound = init.targets().get(s,None)
if not tfound and '*' in s:
matchregex = s.rstrip('$')+'$'
try:
matchregex = re.compile(matchregex, re.IGNORECASE)
except Exception as e:
raise BuildException('Invalid target regular expression "%s": %s'%(matchregex, e))
matches = [t for t in init.targets().values() if matchregex.match(t.name)]
if len(matches) > 1:
print('Found multiple targets matching pattern %s:'%(s), file=stdout)
print(file=stdout)
for m in matches:
print(m.name, file=stdout)
print(file=stdout)
raise BuildException('Target regex must uniquely identify a single target: %s (use tags to specify multiple related targets)'%s)
if matches: return matches[0]
if not tfound: raise BuildException('Unknown target name, target regex or tag name: %s'%s)
return tfound
def _coerceToValidValue(value):
value = BuildInitializationContext.getBuildInitializationContext().expandPropertyValues(value)
if not os.path.isabs(value):
# must absolutize this, as otherwise it might be used from a build
# file in a different location, resulting in the same property
# resolving to different effective values in different places
value = BuildFileLocation(raiseOnError=True).buildDir+'/'+value
value = normpath(value).rstrip('/\\')
if mustExist and not os.path.exists(value):
raise BuildException('Invalid path property value for "%s" - path "%s" does not exist' % (name, value))
return value
def registerOutputDirProperties(*propertyNames):
""" Registers the specified path property name(s) as being an output directory
of this build, meaning that they will be created automatically at the
beginning of the build process, and removed during a global clean.
Typical usage is to call this just after definePathProperty.
"""
init = BuildInitializationContext.getBuildInitializationContext()
if init:
for p in propertyNames:
p = init.getPropertyValue(p)
if not os.path.isabs(p): raise BuildException('Only absolute path properties can be used as output dirs: "%s"'%p)
init.registerOutputDir(normpath(p))
def _defineOption(name, default):
"""
.. private:: For internal use only.
Register an available option and specify its default value.
Called internally from L{propertysupport.defineOption} and does not
need to be called directly
"""
if name in BuildInitializationContext._definedOptions and BuildInitializationContext._definedOptions[
name] != default and 'doctest' not in sys.argv[0]:
raise BuildException('Cannot define option "%s" more than once' %
name)
BuildInitializationContext._definedOptions[name] = default
def priority(self, priority: float):
"""Called by build file authors to configure the priority of this target to encourage it (and its dependencies)
to be built earlier in the process.
The default priority is 0.0
@param priority: a float representing the priority. Higher numbers will be built
first where possible. Cannot be negative.
:return: Returns the same target instance it was called on, to permit fluent calling.
"""
if priority < 0.0:
raise BuildException(
'Target priority cannot be set to a lower number than 0.0')
self.__priority = priority
return self
def _getTargetPathsWithinDir(self, parentDir):
"""Returns a generator yielding the normalized resolved path of all targets
that start with the specified parent directory path. The results will
be in a random order.
@param parentDir: a resolved normalized directory name ending with a slash.
"""
if not isDirPath(parentDir):
raise BuildException(
'Directory paths must have a trailing slash: "%s"' % parentDir)
assert not parentDir.startswith(
'\\\\?'), 'This method is not designed for long paths'
for path in self.__targetPaths:
if path.startswith(parentDir):
yield path
def _copyFile(self, context, src, dest):
if self.getOption('common.fileEncodingDecider')(
context, src) == ExtensionBasedFileEncodingDecider.BINARY:
return super()._copyFile(context, src, dest)
mappers = [
m for m in self.mappers
if m.startFile(context, src, dest) is not False
]
try:
with self.openFile(context, src, 'r', newline='\n') as s:
# newline: for compatibility with existing builds, we don't expand \n to os.linesep (i.e. don't use Python universal newlines)
with self.openFile(context, dest, 'w', newline='\n') as d:
for m in mappers:
x = m.getHeader(context)
if x:
self.__unusedMappers.discard(m)
d.write(x)
for l in s:
for m in mappers:
prev = l
l = m.mapLine(context, l)
if prev != l:
self.__unusedMappers.discard(m)
if None == l:
break
if None != l:
d.write(l)
for m in mappers:
x = m.getFooter(context)
if x:
self.__unusedMappers.discard(m)
d.write(x)
except Exception as ex:
exceptionsuffix = ''
if isinstance(ex, UnicodeDecodeError):
exceptionsuffix = ' due to encoding problem; consider setting the "common.fileEncodingDecider" option'
exceptionsuffix += '; first bad character is: %r' % ex.object[
ex.start:ex.start + 10]
raise BuildException(
f'Failed to perform filtered copy of {src}{exceptionsuffix}',
causedBy=True)
shutil.copymode(src, dest)
assert os.path.exists(dest)
def _resolveItem(self, x, context):
if x == self.DEPENDENCIES: return self.deps.resolve(context)
if x == self.TARGET: x = self.path
if isinstance(x, str): return context.expandPropertyValues(x)
if hasattr(x, 'resolveToString'):
return x.resolveToString(context) # supports Composables too
if isinstance(x, BasePathSet):
result = x.resolve(context)
if len(result) != 1:
raise BuildException(
'PathSet for custom command must resolve to exactly one path not %d (or use joinPaths): %s'
% (len(result), x))
return result[0]
if isinstance(x, ResolvePath): return x.resolve(context, self.baseDir)
if callable(x): return x(context)
raise Exception('Unknown custom command input type %s: %s' %
(x.__class__.__name__, x))
def registerTarget(self, target):
"""
.. private:: For internal use only.
Registers the target with the context.
Called internally from L{basetarget.BaseTarget} and does not need to be called directly.
Will raise an exception if called after the build files have been parsed.
"""
self._initializationCheck()
# don't do anything much with it yet, wait for initialization phase to complete first
if target.name in self._targetsMap:
raise BuildException(
'Duplicate target name "%s" (%s)' %
(target, self._targetsMap[target.name].location),
location=target.location)
self._targetsMap[target.name] = target
self._targetsList.append(target)
self.registerTags(target, target.getTags())
def getOption(self, key, errorIfNone=True, errorIfEmptyString=True):
""" Target classes can call this during `run` or `clean` to get the resolved value of a specified option for
this target, with optional checking to give a friendly error message if the value is an empty string or None.
This is a high-level alternative to reading directly from `self.options`.
This method cannot be used while the build files are still being loaded, only during the execution of the targets.
"""
if hasattr(key, 'optionName'):
key = key.optionName # it's an Option instance
if key not in self.options:
raise Exception(
'Target tried to access an option key that does not exist: %s'
% key)
v = self.options[key]
if (errorIfNone and v == None) or (errorIfEmptyString and v == ''):
raise BuildException(
'This target requires a value to be specified for option "%s" (see basetarget.option or setGlobalOption)'
% key)
return v
