def isIncluded(self, context, path): """ Decides whether the specified path within the archive should be unpacked, based on the include/exclude filters @param path: a relative path within the archive """ if not self.__excludes and not self.__includes: return True if not self.__isResolved: self.__includes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__includes]) self.__excludes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__excludes]) self.__isResolved = True assert '\\' not in path try: path = path.lstrip('/') # first check if it matches an exclude if next( (True for e in self.__excludes if antGlobMatch(e, path)), False): return False if not self.__includes: # include everything return True else: m = next( (i for i in self.__includes if antGlobMatch(i, path)), None) if m: return True else: return False except Exception, e: raise BuildException('FilteredArchiveContents error for %s'%(self), causedBy=True, location=self.__location)
def __init__(self, dir, excludes=None, includes=None): """ @param dir: base directory to search (relative or absolute, may contain ${...} variables). May be a simple string, or a DirGeneratedByTarget to glob under a directory generated as part of the build. @param includes: a list of glob patterns for the files to include (excluding all others) @param excludes: a list of glob patterns to exclude after processing any includes. """ self.__dir = dir self.includes = flatten(includes) self.excludes = flatten(excludes) bad = [ x for x in (self.includes + self.excludes) if ('//' in x or x.startswith('/') or x.startswith('/') or '\\' in x or '${' in x) ] if bad: raise BuildException( 'Invalid includes/excludes pattern in FindPaths - must not contain \\, begin or end with /, or contain substitution variables: "%s"' % bad[0]) if isinstance( dir, basestring ) and '\\' in dir: # avoid silly mistakes, and enforce consistency raise BuildException( 'Invalid base directory for FindPaths - must not contain \\ (always use forward slashes)' ) self.location = BuildFileLocation() self.__lock = threading.Lock() self.__cached = None
def resolveWithDestinations(self, context): children = flatten([ map(lambda s: s.strip(), context.expandPropertyValues(c, expandList=True)) for c in self.__children ]) dir = _resolveDirPath(self.__dir, context, self.__location) result = [] for c in children: if '*' in c: raise BuildException( 'Cannot specify "*" patterns here (consider using FindPaths instead): "%s"' % c, location=self.__location) if os.path.isabs(c): raise BuildException( 'Cannot specify absolute paths here, must be relative (consider using basename): "%s"' % c, location=self.__location) isdir = isDirPath(c) c = os.path.join(context.expandPropertyValues(dir), c) c = os.path.normpath(c.rstrip('\\/' + os.path.sep)) if isdir: c = c + os.path.sep result.append((c, c[len(dir):])) return result
def defineAtomicTargetGroup(*targets): """ The given targets must all be built before anything which depends on any of those targets. Returns the flattened list of targets. """ from buildcontext import getBuildInitializationContext targets = flatten(targets) getBuildInitializationContext().defineAtomicTargetGroup(targets) return targets
def isIncluded(self, context, path): """ Decides whether the specified path within the archive should be unpacked, based on the include/exclude filters @param path: a relative path within the archive """ if not self.__excludes and not self.__includes: return True if not self.__isResolved: self.__includes = flatten([ context.expandPropertyValues(x, expandList=True) for x in self.__includes ]) self.__excludes = flatten([ context.expandPropertyValues(x, expandList=True) for x in self.__excludes ]) self.__isResolved = True assert '\\' not in path try: path = path.lstrip('/') # first check if it matches an exclude if next((True for e in self.__excludes if antGlobMatch(e, path)), False): return False if not self.__includes: # include everything return True else: m = next((i for i in self.__includes if antGlobMatch(i, path)), None) if m: return True else: return False except Exception, e: raise BuildException('FilteredArchiveContents error for %s' % (self), causedBy=True, location=self.__location)
def __init__(self, archivePath, includes=None, excludes=None, destMapper=None): """ @param archivePath: The archive to unpack; either a string or a singleton PathSet @param destMapper: A functor that takes a (context, destPath) where destPath is an archive-relative path (guaranteed to contain / not \\), and returns the desired destination relative path string. The functor should have a deterministic and user-friendly __str__ implementation. @param includes: a list of include patterns (if provided excludes all non-matching files) @param excludes: a list of exclude patterns (processed after includes) """ self.__path = PathSet(archivePath) self.__destMapper = destMapper self.__includes = flatten(includes) self.__excludes = flatten(excludes) self.__location = BuildFileLocation() self.__isResolved = False
def __init__(self, dir, excludes=None, includes=None): """ @param dir: base directory to search (relative or absolute, may contain ${...} variables). May be a simple string, or a DirGeneratedByTarget to glob under a directory generated as part of the build. @param includes: a list of glob patterns for the files to include (excluding all others) @param excludes: a list of glob patterns to exclude after processing any includes. """ self.__dir = dir self.includes = flatten(includes) self.excludes = flatten(excludes) bad = [x for x in (self.includes+self.excludes) if ('//' in x or x.startswith('/') or x.startswith('/') or '\\' in x or '${' in x) ] if bad: raise BuildException('Invalid includes/excludes pattern in FindPaths - must not contain \\, begin or end with /, or contain substitution variables: "%s"'%bad[0]) if isinstance(dir, basestring) and '\\' in dir: # avoid silly mistakes, and enforce consistency raise BuildException('Invalid base directory for FindPaths - must not contain \\ (always use forward slashes)') self.location = BuildFileLocation() self.__lock = threading.Lock() self.__cached = None
def __init__(self, dir, *children): """ @param dir: the base directory, which may include substitution variables. When fully expanded, it is essential that dir ends with a '/'. May be a string or a DirGeneratedByTarget. @param children: strings defining the child files or dirs, which may include ${...} variables but not '*' expansions. Can be specified nested inside tuples or lists if desired. If any of the child strings contains a ${...[]} variable, it will be expanded early and split around the ',' character. """ self.__dir = dir self.__children = flatten(children) self.__location = BuildFileLocation()
def _balanced_split(self, how_many: int, categories: bool): """Split data. Args: how_many (int): Number of test data to have. categories (bool): True if 6 categories else 2 groups. """ uniq_vals = 6 if categories else 2 targets = self._target_categories if categories else self._target_groups splitter = {i: defaultdict(list) for i in range(uniq_vals)} zipped = list(zip(self._data, targets)) random.shuffle(zipped) # inplace for d, t in zipped: # data, target if len(splitter[t]["X_test"]) < how_many: splitter[t]["X_test"].append(d) splitter[t]["y_test"].append(t) else: splitter[t]["X_train"].append(d) splitter[t]["y_train"].append(t) if categories: self.X_train_categories = np.array( flatten([splitter[i]["X_train"] for i in range(uniq_vals)])) self.X_test_categories = np.array( flatten([splitter[i]["X_test"] for i in range(uniq_vals)])) self.y_train_categories = flatten( [splitter[i]["y_train"] for i in range(uniq_vals)]) self.y_test_categories = flatten( [splitter[i]["y_test"] for i in range(uniq_vals)]) else: self.X_train_groups = np.array( flatten([splitter[i]["X_train"] for i in range(uniq_vals)])) self.X_test_groups = np.array( flatten([splitter[i]["X_test"] for i in range(uniq_vals)])) self.y_train_groups = flatten( [splitter[i]["y_train"] for i in range(uniq_vals)]) self.y_test_groups = flatten( [splitter[i]["y_test"] for i in range(uniq_vals)])
def __init__(self, *inputs): """ Construct a PathSet from the specified strings and other pathsets. @param inputs: contains strings, targets and PathSet objects, nested as deeply as you like within lists and tuples. The strings must be absolute paths, or paths relative to the build file where this PathSet is defined, in which case the PathSet must be instantiated during the build file parsing phase (relative paths cannot be used in a PathSet that is instantiated while building or resolving dependencies for a target). Paths may not contain the '*' character, and directory paths must end with an explicit '/'. >>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}').resolveWithDestinations(BaseContext({'x':'X/'}))).replace('\\\\\\\\','/') "[('BUILD_DIR/a', 'a'), ('BUILD_DIR/b/', 'b/'), ('BUILD_DIR/1/2/3/', '3/'), ('BUILD_DIR/4/5/6/', '6/'), ('BUILD_DIR/d/e', 'e'), ('BUILD_DIR/e/f/X/', 'X/')]" >>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}').resolve(BaseContext({'x':'X/'}))).replace('\\\\\\\\','/') "['BUILD_DIR/a', 'BUILD_DIR/b/', 'BUILD_DIR/1/2/3/', 'BUILD_DIR/4/5/6/', 'BUILD_DIR/d/e', 'BUILD_DIR/e/f/X/']" >>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}')) 'PathSet("a", "b/", PathSet("1/2/3/", "4/5/6/"), "d/e", "e/f/${x}")' >>> str(PathSet('a', [[PathSet('1/2/3/', DirGeneratedByTarget('4/5/6/'), '7/8/')]], DirGeneratedByTarget('9/'))._resolveUnderlyingDependencies(BaseContext({}))).replace('\\\\\\\\','/') "['BUILD_DIR/a', 'BUILD_DIR/1/2/3/', 'BUILD_DIR/4/5/6/', 'BUILD_DIR/7/8/', 'BUILD_DIR/9/']" >>> PathSet('a/*').resolve(BaseContext({})) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... BuildException: """ super(BasePathSet, self).__init__() self.contents = flatten(inputs) self.__location = None for x in self.contents: if not (isinstance(x, basestring) or isinstance(x, BasePathSet) or hasattr(x, 'resolveToString')): raise BuildException( 'PathSet may contain only strings, PathSets, Composables, targets and lists - cannot accept %s (%s)' % (x, x.__class__)) self.__location = BuildFileLocation()
def __init__(self, dest, src, *mappers, **kwargs): """ @param dest: the output directory (ending with a "/") or file. Never specify a dest directory that is also written to by another target (e.g. do not specify an output directory here). If you need to write multiple files to the output directory, use separate Copy targets for each. @param src: the input, which may be any combination of strings, PathSets and lists of these. @param mappers: a list of mapper objects that will be used to transform the file, line by line. Can be empty in which case this behaves the same as a normal Copy target. For simple @TOKEN@ replacement see createReplaceDictLineMappers. In addition to per-line changes, it is also possible to specify mappers that add header/footer content to the file. Note that files are read and written in binary mode, so mappers will be dealing directly with platform-specific \\n and \\r characters; python's os.linesep should be used where a platform-neutral newline is required. @param kwargs: Additional parameter: allowUnusedMappers: To avoid build files that accumulate unused cruft or are hard to understand, it by default an an error to include a mapper in this list that is not used, i.e. that does not in any way change the output for any file. We recommend using conditionalization to avoid passing in such mappers e.g. FilteredCopy(target, src, [StringReplaceLineMapper(os.linesep,'\\n') if isWindows() else None]). If this is not practical, set allowUnusedMappers=True to prevent this check. """ self.mappers = [m.getInstance() for m in flatten(mappers)] self.allowUnusedMappers = kwargs.pop('allowUnusedMappers', False) assert not kwargs, 'unknown keyword arg(s): %s' % kwargs super(FilteredCopy, self).__init__( dest, src, implicitDependencies=[m.getDependencies() for m in self.mappers])
def __init__(self, *inputs): """ Construct a PathSet from the specified strings and other pathsets. @param inputs: contains strings, targets and PathSet objects, nested as deeply as you like within lists and tuples. The strings must be absolute paths, or paths relative to the build file where this PathSet is defined, in which case the PathSet must be instantiated during the build file parsing phase (relative paths cannot be used in a PathSet that is instantiated while building or resolving dependencies for a target). Paths may not contain the '*' character, and directory paths must end with an explicit '/'. >>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}').resolveWithDestinations(BaseContext({'x':'X/'}))).replace('\\\\\\\\','/') "[('BUILD_DIR/a', 'a'), ('BUILD_DIR/b/', 'b/'), ('BUILD_DIR/1/2/3/', '3/'), ('BUILD_DIR/4/5/6/', '6/'), ('BUILD_DIR/d/e', 'e'), ('BUILD_DIR/e/f/X/', 'X/')]" >>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}').resolve(BaseContext({'x':'X/'}))).replace('\\\\\\\\','/') "['BUILD_DIR/a', 'BUILD_DIR/b/', 'BUILD_DIR/1/2/3/', 'BUILD_DIR/4/5/6/', 'BUILD_DIR/d/e', 'BUILD_DIR/e/f/X/']" >>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}')) 'PathSet("a", "b/", PathSet("1/2/3/", "4/5/6/"), "d/e", "e/f/${x}")' >>> str(PathSet('a', [[PathSet('1/2/3/', DirGeneratedByTarget('4/5/6/'), '7/8/')]], DirGeneratedByTarget('9/'))._resolveUnderlyingDependencies(BaseContext({}))).replace('\\\\\\\\','/') "['BUILD_DIR/a', 'BUILD_DIR/1/2/3/', 'BUILD_DIR/4/5/6/', 'BUILD_DIR/7/8/', 'BUILD_DIR/9/']" >>> PathSet('a/*').resolve(BaseContext({})) #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): ... BuildException: """ super(BasePathSet, self).__init__() self.contents = flatten(inputs) self.__location = None for x in self.contents: if not (isinstance(x, basestring) or isinstance(x, BasePathSet) or hasattr(x, 'resolveToString')): raise BuildException('PathSet may contain only strings, PathSets, Composables, targets and lists - cannot accept %s (%s)'%(x, x.__class__)) self.__location = BuildFileLocation()
def __init__(self, dest, archives): """ @param dest: the output directory (ending with a "/"). Never specify a dest directory that is also written to by another target (e.g. do not specify a build 'output' directory here). @param archives: the input archives to be unpacked, which may be any combination of strings, PathSets, FilteredArchiveContents and lists of these. If these PathSets include mapping information, this will be used to define where (under the dest directory) each file from within that archive is copied (but cannot be used to change the archive-relative path of each item). For advanced cases, FilteredArchiveContents can be used to provide customized mapping and filtering of the archive contents, including manipulation of the destinations encoded within the archive itself. """ if not dest.endswith('/'): raise BuildException( 'Unpack target destination must be a directory (ending with "/"), not: "%s"' % dest) # NB: we could also support copying in non-archived files into the directory future too # we should preserve the specified order of archives since it may # affect what happens when they contain the same files and must # overwrite each other archives = [ a if (isinstance(a, BasePathSet) or isinstance(a, FilteredArchiveContents)) else PathSet(a) for a in flatten(archives) ] BaseTarget.__init__(self, dest, [(a.getDependency() if isinstance( a, FilteredArchiveContents) else a) for a in archives]) self.archives = archives
def __init__(self, dest, src, *mappers, **kwargs): """ @param dest: the output directory (ending with a "/") or file. Never specify a dest directory that is also written to by another target (e.g. do not specify an output directory here). If you need to write multiple files to the output directory, use separate Copy targets for each. @param src: the input, which may be any combination of strings, PathSets and lists of these. @param mappers: a list of mapper objects that will be used to transform the file, line by line. Can be empty in which case this behaves the same as a normal Copy target. For simple @TOKEN@ replacement see createReplaceDictLineMappers. In addition to per-line changes, it is also possible to specify mappers that add header/footer content to the file. Note that files are read and written in binary mode, so mappers will be dealing directly with platform-specific \\n and \\r characters; python's os.linesep should be used where a platform-neutral newline is required. @param allowUnusedMappers: To avoid build files that accumulate unused cruft or are hard to understand, it by default an an error to include a mapper in this list that is not used, i.e. that does not in any way change the output for any file. We recommend using conditionalization to avoid passing in such mappers e.g. FilteredCopy(target, src, [StringReplaceLineMapper(os.linesep,'\n') if isWindows() else None]). If this is not practical, set allowUnusedMappers=True to prevent this check. """ self.mappers = [m.getInstance() for m in flatten(mappers)] self.allowUnusedMappers = kwargs.pop('allowUnusedMappers', False) assert not kwargs, 'unknown keyword arg(s): %s'%kwargs super(FilteredCopy, self).__init__(dest, src, implicitDependencies=[m.getDependencies() for m in self.mappers])
def resolveWithDestinations(self, context): children = flatten([ map(lambda s: s.strip(), context.expandPropertyValues(c, expandList=True)) for c in self.__children]) dir = _resolveDirPath(self.__dir, context, self.__location) result = [] for c in children: if '*' in c: raise BuildException('Cannot specify "*" patterns here (consider using FindPaths instead): "%s"'%c, location=self.__location) if os.path.isabs(c): raise BuildException('Cannot specify absolute paths here, must be relative (consider using basename): "%s"'%c, location=self.__location) isdir = isDirPath(c) c=os.path.join(context.expandPropertyValues(dir), c) c=os.path.normpath(c.rstrip('\\/'+os.path.sep)) if isdir: c = c+os.path.sep result.append( ( c, c[len(dir):] ) ) return result
def __init__(self, dest, archives): """ @param dest: the output directory (ending with a "/"). Never specify a dest directory that is also written to by another target (e.g. do not specify a build 'output' directory here). @param archives: the input archives to be unpacked, which may be any combination of strings, PathSets, FilteredArchiveContents and lists of these. If these PathSets include mapping information, this will be used to define where (under the dest directory) each file from within that archive is copied (but cannot be used to change the archive-relative path of each item). For advanced cases, FilteredArchiveContents can be used to provide customized mapping and filtering of the archive contents, including manipulation of the destinations encoded within the archive itself. """ if not dest.endswith('/'): raise BuildException('Unpack target destination must be a directory (ending with "/"), not: "%s"'%dest) # NB: we could also support copying in non-archived files into the directory future too # we should preserve the specified order of archives since it may # affect what happens when they contain the same files and must # overwrite each other archives = [a if (isinstance(a, BasePathSet) or isinstance(a, FilteredArchiveContents)) else PathSet(a) for a in flatten(archives)] BaseTarget.__init__(self, dest, [ (a.getDependency() if isinstance(a, FilteredArchiveContents) else a) for a in archives]) self.archives = archives
def __init__(self, patterns): """Do not call this constructor - use L{create} instead of constructing directly. """ patterns = flatten(patterns) # values are [ELEMENTS, ORIGINAL_PATTERNS_INDEX] self.filepatterns = [] self.dirpatterns = [] # with no trailing slash self.origpatterns = [] # original file+dir patterns self.hasStarStarPrefixPattern = False # set to true if there are any file or dir patterns starting **/ # some booleans to save calculations later self.allfiles = False # ** i.e. match all files regardless self.alldirs = False for p in patterns: if not p: continue if '?' in p: # would require some more work (maybe regexes), but a rare case so don't bother raise BuildException( 'Invalid pattern ("?" is not supported at present): %s' % p) if '\\' in p: raise BuildException( 'Invalid pattern (must use forward slashes not backslashes): %s' % p) if p.endswith('**/*'): p = p[:-2] # normalize since this is pointless if '**/*/' in p: # we disallow this because it messes up our ability to decide how many path elements the ** should eat; # breaks cases like >>> antGlobMatch('**/*/f/*.e', 'f/PPP/a/b/c/d/e/PPP/f/foo.e') (ought to be True) # in theory we could solve that case by counting from the back, but the potential presence of # **s later in the pattern would make that a very inefficient operation. # There isn't really a compelling case for supporting this so disable it to avoid people # shooting themselves in the foot raise BuildException( 'Invalid pattern (**/* sequences are not permitted): %s' % p) self.origpatterns.append(p) if p.startswith('**'): self.hasStarStarPrefixPattern = True if p[-1] == '/': if p == '**/': self.alldirs = True elements = [ self.__canonicalizePatternElement(e) for e in p[:-1].split('/') ] self.dirpatterns.append([elements, len(self.origpatterns) - 1]) else: if p == '**': self.allfiles = True elements = [ self.__canonicalizePatternElement(e) for e in p.split('/') ] self.filepatterns.append( [elements, len(self.origpatterns) - 1]) for e in elements: if '**' in e and e != '**': raise BuildException( 'Invalid pattern (pattern elements containing "**" must not have any other characters): %s' % p) if self.allfiles: assert len( self.filepatterns ) == 1, 'No point specifying additional file patterns after adding **' if self.alldirs: assert len( self.dirpatterns ) == 1, 'No point specifying additional directory patterns after adding **/' self.nofiles = self.filepatterns == [] self.nodirs = self.dirpatterns == []