def run(self, context): self.log.info('Cleaning existing files from %s', self.path) deleteDir(self.path) iswindows = IS_WINDOWS for a in self.archives: a_is_filteredarchivecontents = isinstance(a, FilteredArchiveContents) if a_is_filteredarchivecontents: items = [(a.getResolvedPath(context), '')] else: assert isinstance(a, BasePathSet) filteredMembers = None items = a.resolveWithDestinations(context) for (srcAbs, destRel) in items: if destRel and not isDirPath(destRel): destRel = os.path.dirname(destRel) # strip off the zip filename if '..' in destRel: raise Exception('This target does not permit destination paths to contain ".." relative path expressions') try: filesize = os.path.getsize(srcAbs) except Exception: filesize = 0 self.log.info("Unpacking %s (%0.1f MB) to %s", os.path.basename(srcAbs), filesize/1024.0/1024, self.name+destRel) starttime = time.time() with self. __openArchive(srcAbs) as f: mkdir(self.path+destRel) if a_is_filteredarchivecontents and a.hasIncludeExcludeFilters(): fullList = _getnames(f) if not fullList: raise BuildException('No files were found in archive "%s"'%(srcAbs)) filteredMembers = [x for x in fullList if a.isIncluded(context, x)] self.log.info("Unpacking %d of %d members in %s", len(filteredMembers), len(fullList), os.path.basename(srcAbs)) if not filteredMembers: raise BuildException('No files matching the specified include/exclude filters were found in archive "%s": %s'%(srcAbs, a)) if len(filteredMembers)==len(fullList): raise BuildException('No files were excluded from the unpacking operation by the specified filters (check filters are correct): %s'%a) else: filteredMembers = _getnames(f) # NB: some archive types want a list of string members, others want TarInfo objects etc, so # if we support other archive types in future might need to do a bit of work here path = normLongPath(self.path+destRel) for m in filteredMembers: if not isDirPath(m): info = _getinfo(f, m) if a_is_filteredarchivecontents: _setfilename(info, a.mapDestPath(context, _getfilename(info))) if iswindows: _setfilename(info, _getfilename(info).replace('/', '\\')) f.extract(info, path=path) else: # we should create empty directories too if a_is_filteredarchivecontents: m = a.mapDestPath(context, m).rstrip('/') m = path.rstrip('/\\')+'/'+m if iswindows: m = m.replace('/', '\\') mkdir(m) self.log.info("Completed unpacking %s (%0.1f MB) in %0.1f seconds", os.path.basename(srcAbs), filesize/1024.0/1024, (time.time()-starttime))
def __init__(self, target, command=None, dependencies=[], cwd=None, redirectStdOutToTarget=False, env=None, stdout=None, stderr=None, commands=None): BaseTarget.__init__(self, target, dependencies) assert not (command and commands), 'Cannot specify both command= and commands=' self.command = command self.commands = commands self.cwd = cwd self.deps = PathSet(dependencies) self.redirectStdOutToTarget = redirectStdOutToTarget if redirectStdOutToTarget and isDirPath(target): raise BuildException( 'Cannot set redirectStdOutToTarget and specify a directory for the target name - please specify a file instead: %s' % target) self.env = env self.stdout, self.stderr = stdout, stderr if stdout and redirectStdOutToTarget: raise BuildException( 'Cannot set both redirectStdOutToTarget and stdout')
def resolveToString(self, context): """ .. private:: There is usually no need for this to be called other than by the framework. Resolves this target's path and returns as a string. It is acceptable to call this while the build files are still being parsed (before the dependency checking phase), but an error will result if resolution depends on anything that has not yet been defined. """ # implementing this allows targets to be used in Composeable expressions # if there's no explicit parent, default to ${OUTPUT_DIR} to stop # people accidentally writing to their source directories if self.__path is not None: return self.__path # cache it for consistency self.__path = context.getFullPath( self.__path_src, context.getPropertyValue("OUTPUT_DIR")) badchars = '<>:"|?*' # Windows bad characters; it's helpful to stop people using such characters on all OSes too since almost certainly not intended foundbadchars = [ c for c in self.__path[2:] if c in badchars ] # (nb: ignore first 2 chars of absolute path which will necessarily contain a colon on Windows) if foundbadchars: raise BuildException( 'Invalid character(s) "%s" found in target name %s' % (''.join(sorted(list(set(foundbadchars)))), self.__path)) if self.__path.endswith(('.', ' ')): raise BuildException( 'Target name must not end in a "." or " "' ) # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file self.log.debug('Resolved target name %s to canonical path %s', self.name, self.path) return self.__path
def __init__(self, dest, archives): """ @param dest: the output directory (ending with a "/"). Never specify a dest directory that is also written to by another target (e.g. do not specify a build 'output' directory here). @param archives: the input archives to be unpacked, which may be any combination of strings, PathSets, FilteredArchiveContents and lists of these. If these PathSets include mapping information, this will be used to define where (under the dest directory) each file from within that archive is copied (but cannot be used to change the archive-relative path of each item). For advanced cases, FilteredArchiveContents can be used to provide customized mapping and filtering of the archive contents, including manipulation of the destinations encoded within the archive itself. """ if not dest.endswith('/'): raise BuildException('Unpack target destination must be a directory (ending with "/"), not: "%s"'%dest) # NB: we could also support copying in non-archived files into the directory future too # we should preserve the specified order of archives since it may # affect what happens when they contain the same files and must # overwrite each other archives = [a if (isinstance(a, BasePathSet) or isinstance(a, FilteredArchiveContents)) else PathSet(a) for a in flatten(archives)] BaseTarget.__init__(self, dest, [ (a.getDependency() if isinstance(a, FilteredArchiveContents) else a) for a in archives]) self.archives = archives
def isIncluded(self, context, path): """ Decides whether the specified path within the archive should be unpacked, based on the include/exclude filters @param path: a relative path within the archive """ if not self.__excludes and not self.__includes: return True if not self.__isResolved: self.__includes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__includes]) self.__excludes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__excludes]) self.__isResolved = True assert '\\' not in path try: path = path.lstrip('/') # first check if it matches an exclude if next( (True for e in self.__excludes if antGlobMatch(e, path)), False): return False if not self.__includes: # include everything return True else: m = next( (i for i in self.__includes if antGlobMatch(i, path)), None) if m: return True else: return False except Exception as e: raise BuildException('FilteredArchiveContents error for %s'%(self), causedBy=True, location=self.__location)
def __init__(self, extToEncoding={}, default=None): self.extToEncodingDict, self.defaultEncoding = dict(extToEncoding), default # enforce starts with a . to prevent mistakes and allow us to potentially optimize the implementation in future for k in extToEncoding: if not k.startswith('.'): raise BuildException(f'ExtensionBasedFileEncodingDecider extension does not start with the required "." character: "{k}"') self.__cache = None,None self.__stringified = f'ExtensionBasedFileEncodingDecider({self.extToEncodingDict}; default={self.defaultEncoding})'
def setGlobalOption(self, key, value): """ .. private:: For internal use only. Set a global value for an option Called internally from L{propertysupport.setGlobalOption} and does not need to be called directly """ if not key in BuildInitializationContext._definedOptions: raise BuildException( "Cannot specify value for option that has not been defined \"%s\"" % key) if key in self._globalOptions: # for a like-to-like comparison, need to expand any properties (empirically one of them seems to be non-expanded) if self._recursiveExpandProperties( value) != self._recursiveExpandProperties( self._globalOptions[key]): log.warn( "Resetting global option %s to %s at %s; old value was %s", key, self._recursiveExpandProperties(value), BuildFileLocation().getLineString(), self._recursiveExpandProperties(self._globalOptions[key])) else: log.info("Setting global option %s to %s at %s", key, value, BuildFileLocation().getLineString()) self._globalOptions[key] = value
def _coerceToValidValue(value): value = BuildInitializationContext.getBuildInitializationContext().expandPropertyValues(str(value)) if value.lower() == 'true': return True if value.lower() == 'false' or value=='': return False raise BuildException('Invalid property value for "%s" - must be true or false' % (name))
def handleEnd(self, returnCode=None): # linker failures often have no errors but a really useful message in the first warning, so include that in the error message if returnCode and self.getWarnings() and not self.getErrors(): raise BuildException( '%s failed with return code %s (first warning: %s)' % (self._name, returnCode, self.getWarnings()[0])) ProcessOutputHandler.handleEnd(self, returnCode=returnCode)
def handleEnd(self, returnCode=None): """ Called when the process has terminated, to collate any warnings, errors and other messages, and in conjunction with the ``returnCode`` optionally raise an `xpybuild.utils.buildexceptions.BuildException` with a suitable message. The default implementation logs a message summarizing the total number of warnings, then raises a ``BuildException`` if there were any error or (unless ``ignoreReturnCode`` was set) if ``returnCode`` is non-zero. The exception message will contain the first error, or if none the first warning, or failing that, the last line of the output. """ if self._warnings: self._logger.warning('%d warnings during %s', len(self._warnings), self._name) if self._errors: msg = self._errors[0] if len(self._errors)>1: msg = '%d errors, first is: %s'%(len(self._errors), msg) elif returnCode and not self._ignoreReturnCode: msg = '%s failed with return code %s'%(self._name, returnCode) # in case it's relevant, since the return code doesn't provide much to go on if self._warnings: msg += '; no errors reported, first warning was: %s'%self._warnings[0] elif self.getLastOutputLine(): # last-ditch attempt to be useful msg += '; no errors reported, last line was: %s'%self.getLastOutputLine() else: msg += ' and no output generated' else: return raise BuildException(msg)
def call(self, context, args, outputHandler, options, cwd=None, environs=None): try: args = flatten([ context.expandPropertyValues(x, expandList=True) for x in args ]) try: outputHandlerInstance = outputHandler(os.path.basename( args[0]), options=options) except Exception as e: # backwards compatibility for output handlers that don't pass kwargs down outputHandlerInstance = outputHandler(os.path.basename( args[0])) call(args, outputHandler=outputHandlerInstance, cwd=cwd, env=self.getExpandedEnvirons(context, environs), timeout=options['process.timeout']) except BuildException as e: # causedBy is not useful here raise BuildException("%s process failed" % (os.path.basename(args[0])), causedBy=True)
def handleEnd(self, returnCode=None): if returnCode and not self.getErrors() and fatalerrors: # special-case to give a more useful error message tha just the exit code if a dependency is missing raise BuildException( 'Native dependency checking failed: %s' % (fatalerrors[0])) return super(VSDependsHandler, self).handleEnd(returnCode=returnCode)
def enableEnvironmentPropertyOverrides(self, prefix): if not prefix or not prefix.strip(): raise BuildException( 'It is mandatory to specify a prefix for enableEnvironmentPropertyOverrides' ) # read from env now to save doing it repeatedly later for k in os.environ: if k.startswith(prefix): v = os.environ[k] k = k[len(prefix):] if k in self._envPropertyOverrides: # very unlikely, but useful to check raise BuildException( 'Property %s is being read in from the environment in two different ways' % (prefix + k)) self._envPropertyOverrides[k] = v
def getTargetsWithTag(self, tag): """ Returns the list of target objects with the specified tag name (throws BuildException if not defined). """ result = list(self._tags.get(tag, [])) if not result: raise BuildException( 'Tag "%s" is not defined for any target in the build' % tag) return result
def _coerceToValidValue(value): value = BuildInitializationContext.getBuildInitializationContext().expandPropertyValues(value) if value in enumValues: return value # case-insensitive match for e in enumValues: if e.lower()==value.lower(): return e raise BuildException('Invalid property value for "%s" - value "%s" is not one of the allowed enumeration values: %s' % (name, value, enumValues))
def _handle_error(self, target, prefix='Target FAILED'): """ Perform logging for the exception on the stack, and return an array of string to be appended to the global build errors list. target - should be a BaseTarget (not a TargetWrapper) prefix - Prefix of exception, describing what we were doing at the time """ e = sys.exc_info()[1] logged = False if not isinstance(e, BuildException): if not isinstance(e, (EnvironmentError)): # most problems should be wrapped as BuildException already; let's make sure we always # get an ERROR-level message for things like syntax errors etc #log.exception('%s: unexpected (non-build) exception in %s'%(prefix, target)) #logged = True pass # this duplicates the stack trace we get at ERROR level from toMultiLineString e = BuildException('%s due to %s' % (prefix, e.__class__.__name__), causedBy=True) if not logged and log.isEnabledFor( logging.DEBUG ): # make sure the stack trace is at least available at debug log.debug('Handling error: %s', traceback.format_exc()) # one-line summary (if in teamcity mode, we'd use teamcity syntax to log this) #log.error('%s: %s', prefix, e.toSingleLineString(target)) # also useful to have the full stack trace, but only at INFO level #log.info('%s (details): %s\n', prefix, e.toMultiLineString(target, includeStack=True)) # one-line summary (if in teamcity mode, we'd use teamcity syntax to log this) #log.error('%s: %s', prefix, e.toSingleLineString(target)) # also useful to have the full stack trace, but only at INFO level log.error('%s: %s\n', prefix, e.toMultiLineString(target, includeStack=True), extra=e.getLoggerExtraArgDict(target)) return [e.toSingleLineString(target)]
def run(self, context): self.__unusedMappers = set(self.mappers) super(FilteredCopy, self).run(context) if self.__unusedMappers and not self.allowUnusedMappers: # a useful sanity check, to ensure we're replacing what we think we're replacing, and also that we don't have unused clutter in the build files raise BuildException( 'Some of the specified mappers did not get used at all during the copy (to avoid confusion, mappers that do not change the output in any way are not permitted): %s' % (', '.join( [m.getDescription(context) for m in self.__unusedMappers])))
def _mergeListOfOptionDicts(self, dicts, target=None): # creates a new dictionary # dicts is a list of dictionaries # target is used just for error reporting, if available fulloptions = {} for source in dicts: if source is None: continue for key in source: try: if key not in BuildInitializationContext._definedOptions: raise BuildException("Unknown option %s" % key) fulloptions[key] = self._recursiveExpandProperties( source[key]) except BuildException as e: raise BuildException( 'Failed to resolve option "%s"' % key, location=target.location if target else None, causedBy=True) from None return fulloptions
def checkForNonTargetDependenciesUnderOutputDirs(self): """ Iterate over the dependencies that are not targets and return the name of one that's under an output dir (suggests a missing target dep), or None. """ ctx = self.__scheduler.context for dpath, flags, pathset in self.__nontargetdeps: if ctx.isPathWithinOutputDir(dpath): raise BuildException('Target %s depends on output %s which is implicitly created by some other directory target - please use DirGeneratedByTarget to explicitly name the directory target that it depends on'%(self, dpath), location=self.target.location) # e.g. FindPaths(DirGeneratedByTarget('${OUTPUT_DIR}/foo/')+'bar/') # or similar
def run(self, context): mkdir(os.path.dirname(self.path)) alreadyDone = set() with zipfile.ZipFile(normLongPath(self.path), 'w') as output: for (f, o) in self.inputs.resolveWithDestinations(context): # if we don't check for duplicate entries we'll end up creating an invalid zip if o in alreadyDone: dupsrc = ['"%s"'%src for (src, dest) in self.inputs.resolveWithDestinations(context) if dest == o] raise BuildException('Duplicate zip entry "%s" from: %s'%(o, ', '.join(dupsrc))) alreadyDone.add(o) # can't compress directory entries! (it messes up Java) output.write(normLongPath(f).rstrip('/\\'), o, zipfile.ZIP_STORED if isDirPath(f) else zipfile.ZIP_DEFLATED)
def lookupTarget(s): tfound = init.targets().get(s,None) if not tfound and '*' in s: matchregex = s.rstrip('$')+'$' try: matchregex = re.compile(matchregex, re.IGNORECASE) except Exception as e: raise BuildException('Invalid target regular expression "%s": %s'%(matchregex, e)) matches = [t for t in init.targets().values() if matchregex.match(t.name)] if len(matches) > 1: print('Found multiple targets matching pattern %s:'%(s), file=stdout) print(file=stdout) for m in matches: print(m.name, file=stdout) print(file=stdout) raise BuildException('Target regex must uniquely identify a single target: %s (use tags to specify multiple related targets)'%s) if matches: return matches[0] if not tfound: raise BuildException('Unknown target name, target regex or tag name: %s'%s) return tfound
def _coerceToValidValue(value): value = BuildInitializationContext.getBuildInitializationContext().expandPropertyValues(value) if not os.path.isabs(value): # must absolutize this, as otherwise it might be used from a build # file in a different location, resulting in the same property # resolving to different effective values in different places value = BuildFileLocation(raiseOnError=True).buildDir+'/'+value value = normpath(value).rstrip('/\\') if mustExist and not os.path.exists(value): raise BuildException('Invalid path property value for "%s" - path "%s" does not exist' % (name, value)) return value
def registerOutputDirProperties(*propertyNames): """ Registers the specified path property name(s) as being an output directory of this build, meaning that they will be created automatically at the beginning of the build process, and removed during a global clean. Typical usage is to call this just after definePathProperty. """ init = BuildInitializationContext.getBuildInitializationContext() if init: for p in propertyNames: p = init.getPropertyValue(p) if not os.path.isabs(p): raise BuildException('Only absolute path properties can be used as output dirs: "%s"'%p) init.registerOutputDir(normpath(p))
def _defineOption(name, default): """ .. private:: For internal use only. Register an available option and specify its default value. Called internally from L{propertysupport.defineOption} and does not need to be called directly """ if name in BuildInitializationContext._definedOptions and BuildInitializationContext._definedOptions[ name] != default and 'doctest' not in sys.argv[0]: raise BuildException('Cannot define option "%s" more than once' % name) BuildInitializationContext._definedOptions[name] = default
def priority(self, priority: float): """Called by build file authors to configure the priority of this target to encourage it (and its dependencies) to be built earlier in the process. The default priority is 0.0 @param priority: a float representing the priority. Higher numbers will be built first where possible. Cannot be negative. :return: Returns the same target instance it was called on, to permit fluent calling. """ if priority < 0.0: raise BuildException( 'Target priority cannot be set to a lower number than 0.0') self.__priority = priority return self
def _getTargetPathsWithinDir(self, parentDir): """Returns a generator yielding the normalized resolved path of all targets that start with the specified parent directory path. The results will be in a random order. @param parentDir: a resolved normalized directory name ending with a slash. """ if not isDirPath(parentDir): raise BuildException( 'Directory paths must have a trailing slash: "%s"' % parentDir) assert not parentDir.startswith( '\\\\?'), 'This method is not designed for long paths' for path in self.__targetPaths: if path.startswith(parentDir): yield path
def _copyFile(self, context, src, dest): if self.getOption('common.fileEncodingDecider')( context, src) == ExtensionBasedFileEncodingDecider.BINARY: return super()._copyFile(context, src, dest) mappers = [ m for m in self.mappers if m.startFile(context, src, dest) is not False ] try: with self.openFile(context, src, 'r', newline='\n') as s: # newline: for compatibility with existing builds, we don't expand \n to os.linesep (i.e. don't use Python universal newlines) with self.openFile(context, dest, 'w', newline='\n') as d: for m in mappers: x = m.getHeader(context) if x: self.__unusedMappers.discard(m) d.write(x) for l in s: for m in mappers: prev = l l = m.mapLine(context, l) if prev != l: self.__unusedMappers.discard(m) if None == l: break if None != l: d.write(l) for m in mappers: x = m.getFooter(context) if x: self.__unusedMappers.discard(m) d.write(x) except Exception as ex: exceptionsuffix = '' if isinstance(ex, UnicodeDecodeError): exceptionsuffix = ' due to encoding problem; consider setting the "common.fileEncodingDecider" option' exceptionsuffix += '; first bad character is: %r' % ex.object[ ex.start:ex.start + 10] raise BuildException( f'Failed to perform filtered copy of {src}{exceptionsuffix}', causedBy=True) shutil.copymode(src, dest) assert os.path.exists(dest)
def _resolveItem(self, x, context): if x == self.DEPENDENCIES: return self.deps.resolve(context) if x == self.TARGET: x = self.path if isinstance(x, str): return context.expandPropertyValues(x) if hasattr(x, 'resolveToString'): return x.resolveToString(context) # supports Composables too if isinstance(x, BasePathSet): result = x.resolve(context) if len(result) != 1: raise BuildException( 'PathSet for custom command must resolve to exactly one path not %d (or use joinPaths): %s' % (len(result), x)) return result[0] if isinstance(x, ResolvePath): return x.resolve(context, self.baseDir) if callable(x): return x(context) raise Exception('Unknown custom command input type %s: %s' % (x.__class__.__name__, x))
def registerTarget(self, target): """ .. private:: For internal use only. Registers the target with the context. Called internally from L{basetarget.BaseTarget} and does not need to be called directly. Will raise an exception if called after the build files have been parsed. """ self._initializationCheck() # don't do anything much with it yet, wait for initialization phase to complete first if target.name in self._targetsMap: raise BuildException( 'Duplicate target name "%s" (%s)' % (target, self._targetsMap[target.name].location), location=target.location) self._targetsMap[target.name] = target self._targetsList.append(target) self.registerTags(target, target.getTags())
def getOption(self, key, errorIfNone=True, errorIfEmptyString=True): """ Target classes can call this during `run` or `clean` to get the resolved value of a specified option for this target, with optional checking to give a friendly error message if the value is an empty string or None. This is a high-level alternative to reading directly from `self.options`. This method cannot be used while the build files are still being loaded, only during the execution of the targets. """ if hasattr(key, 'optionName'): key = key.optionName # it's an Option instance if key not in self.options: raise Exception( 'Target tried to access an option key that does not exist: %s' % key) v = self.options[key] if (errorIfNone and v == None) or (errorIfEmptyString and v == ''): raise BuildException( 'This target requires a value to be specified for option "%s" (see basetarget.option or setGlobalOption)' % key) return v