def isIncluded(self, context, path):
""" Decides whether the specified path within the archive should be
unpacked, based on the include/exclude filters
@param path: a relative path within the archive
"""
if not self.__excludes and not self.__includes: return True
if not self.__isResolved:
self.__includes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__includes])
self.__excludes = flatten([context.expandPropertyValues(x, expandList=True) for x in self.__excludes])
self.__isResolved = True
assert '\\' not in path
try:
path = path.lstrip('/')
# first check if it matches an exclude
if next( (True for e in self.__excludes if antGlobMatch(e, path)), False): return False
if not self.__includes: # include everything
return True
else:
m = next( (i for i in self.__includes if antGlobMatch(i, path)), None)
if m:
return True
else:
return False
except Exception, e:
raise BuildException('FilteredArchiveContents error for %s'%(self), causedBy=True, location=self.__location)
def __init__(self, dir, excludes=None, includes=None):
"""
@param dir: base directory to search (relative or absolute, may contain ${...} variables).
May be a simple string, or a DirGeneratedByTarget to glob under a
directory generated as part of the build.
@param includes: a list of glob patterns for the files to include (excluding all others)
@param excludes: a list of glob patterns to exclude after processing any includes.
"""
self.__dir = dir
self.includes = flatten(includes)
self.excludes = flatten(excludes)
bad = [
x for x in (self.includes + self.excludes)
if ('//' in x or x.startswith('/') or x.startswith('/')
or '\\' in x or '${' in x)
]
if bad:
raise BuildException(
'Invalid includes/excludes pattern in FindPaths - must not contain \\, begin or end with /, or contain substitution variables: "%s"'
% bad[0])
if isinstance(
dir, basestring
) and '\\' in dir: # avoid silly mistakes, and enforce consistency
raise BuildException(
'Invalid base directory for FindPaths - must not contain \\ (always use forward slashes)'
)
self.location = BuildFileLocation()
self.__lock = threading.Lock()
self.__cached = None
def resolveWithDestinations(self, context):
children = flatten([
map(lambda s: s.strip(),
context.expandPropertyValues(c, expandList=True))
for c in self.__children
])
dir = _resolveDirPath(self.__dir, context, self.__location)
result = []
for c in children:
if '*' in c:
raise BuildException(
'Cannot specify "*" patterns here (consider using FindPaths instead): "%s"'
% c,
location=self.__location)
if os.path.isabs(c):
raise BuildException(
'Cannot specify absolute paths here, must be relative (consider using basename): "%s"'
% c,
location=self.__location)
isdir = isDirPath(c)
c = os.path.join(context.expandPropertyValues(dir), c)
c = os.path.normpath(c.rstrip('\\/' + os.path.sep))
if isdir: c = c + os.path.sep
result.append((c, c[len(dir):]))
return result
def defineAtomicTargetGroup(*targets):
""" The given targets must all be built before anything which depends on any of those targets.
Returns the flattened list of targets.
"""
from buildcontext import getBuildInitializationContext
targets = flatten(targets)
getBuildInitializationContext().defineAtomicTargetGroup(targets)
return targets
def isIncluded(self, context, path):
""" Decides whether the specified path within the archive should be
unpacked, based on the include/exclude filters
@param path: a relative path within the archive
"""
if not self.__excludes and not self.__includes: return True
if not self.__isResolved:
self.__includes = flatten([
context.expandPropertyValues(x, expandList=True)
for x in self.__includes
])
self.__excludes = flatten([
context.expandPropertyValues(x, expandList=True)
for x in self.__excludes
])
self.__isResolved = True
assert '\\' not in path
try:
path = path.lstrip('/')
# first check if it matches an exclude
if next((True for e in self.__excludes if antGlobMatch(e, path)),
False):
return False
if not self.__includes: # include everything
return True
else:
m = next((i for i in self.__includes if antGlobMatch(i, path)),
None)
if m:
return True
else:
return False
except Exception, e:
raise BuildException('FilteredArchiveContents error for %s' %
(self),
causedBy=True,
location=self.__location)
def __init__(self, archivePath, includes=None, excludes=None, destMapper=None): """ @param archivePath: The archive to unpack; either a string or a singleton PathSet @param destMapper: A functor that takes a (context, destPath) where destPath is an archive-relative path (guaranteed to contain / not \\), and returns the desired destination relative path string. The functor should have a deterministic and user-friendly __str__ implementation. @param includes: a list of include patterns (if provided excludes all non-matching files) @param excludes: a list of exclude patterns (processed after includes) """ self.__path = PathSet(archivePath) self.__destMapper = destMapper self.__includes = flatten(includes) self.__excludes = flatten(excludes) self.__location = BuildFileLocation() self.__isResolved = False
def __init__(self, dir, excludes=None, includes=None):
"""
@param dir: base directory to search (relative or absolute, may contain ${...} variables).
May be a simple string, or a DirGeneratedByTarget to glob under a
directory generated as part of the build.
@param includes: a list of glob patterns for the files to include (excluding all others)
@param excludes: a list of glob patterns to exclude after processing any includes.
"""
self.__dir = dir
self.includes = flatten(includes)
self.excludes = flatten(excludes)
bad = [x for x in (self.includes+self.excludes) if ('//' in x or x.startswith('/') or x.startswith('/') or '\\' in x or '${' in x) ]
if bad:
raise BuildException('Invalid includes/excludes pattern in FindPaths - must not contain \\, begin or end with /, or contain substitution variables: "%s"'%bad[0])
if isinstance(dir, basestring) and '\\' in dir: # avoid silly mistakes, and enforce consistency
raise BuildException('Invalid base directory for FindPaths - must not contain \\ (always use forward slashes)')
self.location = BuildFileLocation()
self.__lock = threading.Lock()
self.__cached = None
def __init__(self,
archivePath,
includes=None,
excludes=None,
destMapper=None):
"""
@param archivePath: The archive to unpack; either a string or a singleton PathSet
@param destMapper: A functor that takes a (context, destPath) where destPath
is an archive-relative path (guaranteed to contain / not \\), and
returns the desired destination relative path string.
The functor should have a deterministic and
user-friendly __str__ implementation.
@param includes: a list of include patterns (if provided excludes all non-matching files)
@param excludes: a list of exclude patterns (processed after includes)
"""
self.__path = PathSet(archivePath)
self.__destMapper = destMapper
self.__includes = flatten(includes)
self.__excludes = flatten(excludes)
self.__location = BuildFileLocation()
self.__isResolved = False
def __init__(self, dir, *children):
"""
@param dir: the base directory, which may include substitution variables.
When fully expanded, it is essential that dir ends with a '/'.
May be a string or a DirGeneratedByTarget.
@param children: strings defining the child files or dirs, which may
include ${...} variables but not '*' expansions. Can be specified
nested inside tuples or lists if desired. If any of the child
strings contains a ${...[]} variable, it will be expanded
early and split around the ',' character.
"""
self.__dir = dir
self.__children = flatten(children)
self.__location = BuildFileLocation()
def __init__(self, dir, *children):
"""
@param dir: the base directory, which may include substitution variables.
When fully expanded, it is essential that dir ends with a '/'.
May be a string or a DirGeneratedByTarget.
@param children: strings defining the child files or dirs, which may
include ${...} variables but not '*' expansions. Can be specified
nested inside tuples or lists if desired. If any of the child
strings contains a ${...[]} variable, it will be expanded
early and split around the ',' character.
"""
self.__dir = dir
self.__children = flatten(children)
self.__location = BuildFileLocation()
def _balanced_split(self, how_many: int, categories: bool):
"""Split data.
Args:
how_many (int): Number of test data to have.
categories (bool): True if 6 categories else 2 groups.
"""
uniq_vals = 6 if categories else 2
targets = self._target_categories if categories else self._target_groups
splitter = {i: defaultdict(list) for i in range(uniq_vals)}
zipped = list(zip(self._data, targets))
random.shuffle(zipped) # inplace
for d, t in zipped: # data, target
if len(splitter[t]["X_test"]) < how_many:
splitter[t]["X_test"].append(d)
splitter[t]["y_test"].append(t)
else:
splitter[t]["X_train"].append(d)
splitter[t]["y_train"].append(t)
if categories:
self.X_train_categories = np.array(
flatten([splitter[i]["X_train"] for i in range(uniq_vals)]))
self.X_test_categories = np.array(
flatten([splitter[i]["X_test"] for i in range(uniq_vals)]))
self.y_train_categories = flatten(
[splitter[i]["y_train"] for i in range(uniq_vals)])
self.y_test_categories = flatten(
[splitter[i]["y_test"] for i in range(uniq_vals)])
else:
self.X_train_groups = np.array(
flatten([splitter[i]["X_train"] for i in range(uniq_vals)]))
self.X_test_groups = np.array(
flatten([splitter[i]["X_test"] for i in range(uniq_vals)]))
self.y_train_groups = flatten(
[splitter[i]["y_train"] for i in range(uniq_vals)])
self.y_test_groups = flatten(
[splitter[i]["y_test"] for i in range(uniq_vals)])
def __init__(self, *inputs):
"""
Construct a PathSet from the specified strings and other pathsets.
@param inputs: contains strings, targets and PathSet objects, nested as deeply as
you like within lists and tuples.
The strings must be absolute paths, or paths relative to the build file
where this PathSet is defined, in which case the PathSet must be
instantiated during the build file parsing phase (relative paths cannot
be used in a PathSet that is instantiated while building or resolving
dependencies for a target).
Paths may not contain the '*' character, and directory
paths must end with an explicit '/'.
>>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}').resolveWithDestinations(BaseContext({'x':'X/'}))).replace('\\\\\\\\','/')
"[('BUILD_DIR/a', 'a'), ('BUILD_DIR/b/', 'b/'), ('BUILD_DIR/1/2/3/', '3/'), ('BUILD_DIR/4/5/6/', '6/'), ('BUILD_DIR/d/e', 'e'), ('BUILD_DIR/e/f/X/', 'X/')]"
>>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}').resolve(BaseContext({'x':'X/'}))).replace('\\\\\\\\','/')
"['BUILD_DIR/a', 'BUILD_DIR/b/', 'BUILD_DIR/1/2/3/', 'BUILD_DIR/4/5/6/', 'BUILD_DIR/d/e', 'BUILD_DIR/e/f/X/']"
>>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}'))
'PathSet("a", "b/", PathSet("1/2/3/", "4/5/6/"), "d/e", "e/f/${x}")'
>>> str(PathSet('a', [[PathSet('1/2/3/', DirGeneratedByTarget('4/5/6/'), '7/8/')]], DirGeneratedByTarget('9/'))._resolveUnderlyingDependencies(BaseContext({}))).replace('\\\\\\\\','/')
"['BUILD_DIR/a', 'BUILD_DIR/1/2/3/', 'BUILD_DIR/4/5/6/', 'BUILD_DIR/7/8/', 'BUILD_DIR/9/']"
>>> PathSet('a/*').resolve(BaseContext({})) #doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
BuildException:
"""
super(BasePathSet, self).__init__()
self.contents = flatten(inputs)
self.__location = None
for x in self.contents:
if not (isinstance(x, basestring) or isinstance(x, BasePathSet)
or hasattr(x, 'resolveToString')):
raise BuildException(
'PathSet may contain only strings, PathSets, Composables, targets and lists - cannot accept %s (%s)'
% (x, x.__class__))
self.__location = BuildFileLocation()
def __init__(self, dest, src, *mappers, **kwargs):
"""
@param dest: the output directory (ending with a "/") or file. Never
specify a dest directory that is also written to by another
target (e.g. do not specify an output directory here). If you need
to write multiple files to the output directory, use separate Copy
targets for each.
@param src: the input, which may be any combination of strings, PathSets and
lists of these.
@param mappers: a list of mapper objects that will be used to transform
the file, line by line. Can be empty in which case this behaves the same
as a normal Copy target.
For simple @TOKEN@ replacement see createReplaceDictLineMappers.
In addition to per-line changes, it is also possible to specify
mappers that add header/footer content to the file.
Note that files are read and written in binary mode, so mappers
will be dealing directly with platform-specific \\n and \\r
characters; python's os.linesep should be used where a
platform-neutral newline is required.
@param kwargs: Additional parameter: allowUnusedMappers:
To avoid build files that accumulate unused
cruft or are hard to understand, it by default an an error to include a
mapper in this list that is not used, i.e. that does not in any way
change the output for any file. We recommend using conditionalization
to avoid passing in such mappers e.g.
FilteredCopy(target, src, [StringReplaceLineMapper(os.linesep,'\\n') if isWindows() else None]).
If this is not practical, set allowUnusedMappers=True to prevent this
check.
"""
self.mappers = [m.getInstance() for m in flatten(mappers)]
self.allowUnusedMappers = kwargs.pop('allowUnusedMappers', False)
assert not kwargs, 'unknown keyword arg(s): %s' % kwargs
super(FilteredCopy, self).__init__(
dest,
src,
implicitDependencies=[m.getDependencies() for m in self.mappers])
def __init__(self, *inputs):
"""
Construct a PathSet from the specified strings and other pathsets.
@param inputs: contains strings, targets and PathSet objects, nested as deeply as
you like within lists and tuples.
The strings must be absolute paths, or paths relative to the build file
where this PathSet is defined, in which case the PathSet must be
instantiated during the build file parsing phase (relative paths cannot
be used in a PathSet that is instantiated while building or resolving
dependencies for a target).
Paths may not contain the '*' character, and directory
paths must end with an explicit '/'.
>>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}').resolveWithDestinations(BaseContext({'x':'X/'}))).replace('\\\\\\\\','/')
"[('BUILD_DIR/a', 'a'), ('BUILD_DIR/b/', 'b/'), ('BUILD_DIR/1/2/3/', '3/'), ('BUILD_DIR/4/5/6/', '6/'), ('BUILD_DIR/d/e', 'e'), ('BUILD_DIR/e/f/X/', 'X/')]"
>>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}').resolve(BaseContext({'x':'X/'}))).replace('\\\\\\\\','/')
"['BUILD_DIR/a', 'BUILD_DIR/b/', 'BUILD_DIR/1/2/3/', 'BUILD_DIR/4/5/6/', 'BUILD_DIR/d/e', 'BUILD_DIR/e/f/X/']"
>>> str(PathSet('a', [('b/', PathSet('1/2/3/', '4/5/6/'), ['d/e'])], 'e/f/${x}'))
'PathSet("a", "b/", PathSet("1/2/3/", "4/5/6/"), "d/e", "e/f/${x}")'
>>> str(PathSet('a', [[PathSet('1/2/3/', DirGeneratedByTarget('4/5/6/'), '7/8/')]], DirGeneratedByTarget('9/'))._resolveUnderlyingDependencies(BaseContext({}))).replace('\\\\\\\\','/')
"['BUILD_DIR/a', 'BUILD_DIR/1/2/3/', 'BUILD_DIR/4/5/6/', 'BUILD_DIR/7/8/', 'BUILD_DIR/9/']"
>>> PathSet('a/*').resolve(BaseContext({})) #doctest: +IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
BuildException:
"""
super(BasePathSet, self).__init__()
self.contents = flatten(inputs)
self.__location = None
for x in self.contents:
if not (isinstance(x, basestring) or isinstance(x, BasePathSet) or hasattr(x, 'resolveToString')):
raise BuildException('PathSet may contain only strings, PathSets, Composables, targets and lists - cannot accept %s (%s)'%(x, x.__class__))
self.__location = BuildFileLocation()
def __init__(self, dest, archives):
"""
@param dest: the output directory (ending with a "/"). Never
specify a dest directory that is also written to by another
target (e.g. do not specify a build 'output' directory here).
@param archives: the input archives to be unpacked, which may be any
combination of strings, PathSets, FilteredArchiveContents and lists of these.
If these PathSets include mapping information, this
will be used to define where (under the dest directory) each
file from within that archive is copied (but cannot be used to
change the archive-relative path of each item).
For advanced cases, FilteredArchiveContents can be used to provide
customized mapping and filtering of the archive contents,
including manipulation of the destinations encoded within the
archive itself.
"""
if not dest.endswith('/'):
raise BuildException(
'Unpack target destination must be a directory (ending with "/"), not: "%s"'
% dest)
# NB: we could also support copying in non-archived files into the directory future too
# we should preserve the specified order of archives since it may
# affect what happens when they contain the same files and must
# overwrite each other
archives = [
a if (isinstance(a, BasePathSet)
or isinstance(a, FilteredArchiveContents)) else PathSet(a)
for a in flatten(archives)
]
BaseTarget.__init__(self, dest, [(a.getDependency() if isinstance(
a, FilteredArchiveContents) else a) for a in archives])
self.archives = archives
def __init__(self, dest, src, *mappers, **kwargs):
"""
@param dest: the output directory (ending with a "/") or file. Never
specify a dest directory that is also written to by another
target (e.g. do not specify an output directory here). If you need
to write multiple files to the output directory, use separate Copy
targets for each.
@param src: the input, which may be any combination of strings, PathSets and
lists of these.
@param mappers: a list of mapper objects that will be used to transform
the file, line by line. Can be empty in which case this behaves the same
as a normal Copy target.
For simple @TOKEN@ replacement see createReplaceDictLineMappers.
In addition to per-line changes, it is also possible to specify
mappers that add header/footer content to the file.
Note that files are read and written in binary mode, so mappers
will be dealing directly with platform-specific \\n and \\r
characters; python's os.linesep should be used where a
platform-neutral newline is required.
@param allowUnusedMappers: To avoid build files that accumulate unused
cruft or are hard to understand, it by default an an error to include a
mapper in this list that is not used, i.e. that does not in any way
change the output for any file. We recommend using conditionalization
to avoid passing in such mappers e.g.
FilteredCopy(target, src, [StringReplaceLineMapper(os.linesep,'\n') if isWindows() else None]).
If this is not practical, set allowUnusedMappers=True to prevent this
check.
"""
self.mappers = [m.getInstance() for m in flatten(mappers)]
self.allowUnusedMappers = kwargs.pop('allowUnusedMappers', False)
assert not kwargs, 'unknown keyword arg(s): %s'%kwargs
super(FilteredCopy, self).__init__(dest, src, implicitDependencies=[m.getDependencies() for m in self.mappers])
def resolveWithDestinations(self, context):
children = flatten([
map(lambda s: s.strip(), context.expandPropertyValues(c, expandList=True))
for c in self.__children])
dir = _resolveDirPath(self.__dir, context, self.__location)
result = []
for c in children:
if '*' in c:
raise BuildException('Cannot specify "*" patterns here (consider using FindPaths instead): "%s"'%c, location=self.__location)
if os.path.isabs(c):
raise BuildException('Cannot specify absolute paths here, must be relative (consider using basename): "%s"'%c, location=self.__location)
isdir = isDirPath(c)
c=os.path.join(context.expandPropertyValues(dir), c)
c=os.path.normpath(c.rstrip('\\/'+os.path.sep))
if isdir: c = c+os.path.sep
result.append( ( c, c[len(dir):] ) )
return result
def __init__(self, dest, archives):
"""
@param dest: the output directory (ending with a "/"). Never
specify a dest directory that is also written to by another
target (e.g. do not specify a build 'output' directory here).
@param archives: the input archives to be unpacked, which may be any
combination of strings, PathSets, FilteredArchiveContents and lists of these.
If these PathSets include mapping information, this
will be used to define where (under the dest directory) each
file from within that archive is copied (but cannot be used to
change the archive-relative path of each item).
For advanced cases, FilteredArchiveContents can be used to provide
customized mapping and filtering of the archive contents,
including manipulation of the destinations encoded within the
archive itself.
"""
if not dest.endswith('/'): raise BuildException('Unpack target destination must be a directory (ending with "/"), not: "%s"'%dest)
# NB: we could also support copying in non-archived files into the directory future too
# we should preserve the specified order of archives since it may
# affect what happens when they contain the same files and must
# overwrite each other
archives = [a if (isinstance(a, BasePathSet) or isinstance(a, FilteredArchiveContents)) else PathSet(a) for a in flatten(archives)]
BaseTarget.__init__(self, dest, [
(a.getDependency() if isinstance(a, FilteredArchiveContents) else a)
for a in archives])
self.archives = archives
def __init__(self, patterns):
"""Do not call this constructor - use L{create} instead of constructing directly.
"""
patterns = flatten(patterns)
# values are [ELEMENTS, ORIGINAL_PATTERNS_INDEX]
self.filepatterns = []
self.dirpatterns = [] # with no trailing slash
self.origpatterns = [] # original file+dir patterns
self.hasStarStarPrefixPattern = False # set to true if there are any file or dir patterns starting **/
# some booleans to save calculations later
self.allfiles = False # ** i.e. match all files regardless
self.alldirs = False
for p in patterns:
if not p: continue
if '?' in p:
# would require some more work (maybe regexes), but a rare case so don't bother
raise BuildException(
'Invalid pattern ("?" is not supported at present): %s' %
p)
if '\\' in p:
raise BuildException(
'Invalid pattern (must use forward slashes not backslashes): %s'
% p)
if p.endswith('**/*'):
p = p[:-2] # normalize since this is pointless
if '**/*/' in p:
# we disallow this because it messes up our ability to decide how many path elements the ** should eat;
# breaks cases like >>> antGlobMatch('**/*/f/*.e', 'f/PPP/a/b/c/d/e/PPP/f/foo.e') (ought to be True)
# in theory we could solve that case by counting from the back, but the potential presence of
# **s later in the pattern would make that a very inefficient operation.
# There isn't really a compelling case for supporting this so disable it to avoid people
# shooting themselves in the foot
raise BuildException(
'Invalid pattern (**/* sequences are not permitted): %s' %
p)
self.origpatterns.append(p)
if p.startswith('**'): self.hasStarStarPrefixPattern = True
if p[-1] == '/':
if p == '**/': self.alldirs = True
elements = [
self.__canonicalizePatternElement(e)
for e in p[:-1].split('/')
]
self.dirpatterns.append([elements, len(self.origpatterns) - 1])
else:
if p == '**': self.allfiles = True
elements = [
self.__canonicalizePatternElement(e) for e in p.split('/')
]
self.filepatterns.append(
[elements, len(self.origpatterns) - 1])
for e in elements:
if '**' in e and e != '**':
raise BuildException(
'Invalid pattern (pattern elements containing "**" must not have any other characters): %s'
% p)
if self.allfiles:
assert len(
self.filepatterns
) == 1, 'No point specifying additional file patterns after adding **'
if self.alldirs:
assert len(
self.dirpatterns
) == 1, 'No point specifying additional directory patterns after adding **/'
self.nofiles = self.filepatterns == []
self.nodirs = self.dirpatterns == []
