def expression(self):
"""The underlying expression of this file list.
Returns
-------
expression : str
The underlying expression of this source.
"""
if self._expression is None:
self._expression = te.detect(self.file_list)
return self._expression
def _expression_and_file_list(expression=None, file_list=None):
if isinstance(expression, te.Expression) or expression is None:
pass
elif fu.is_directory(expression):
if file_list is None:
file_list = glob.glob(fu.join(expression, '*'))
expression = te.detect(file_list)
elif isinstance(expression, (str, list)):
expression = te.Expression(expression)
else:
raise ValueError('Expression %r is not valid!' % expression)
if file_list is None:
if expression is None:
raise ValueError(
'Either expression or file_list need to be specified!')
file_list = glob.glob(expression.glob())
elif isinstance(file_list, list):
if expression is None:
expression = te.detect(file_list)
else:
raise ValueError('The file_list %r is not value!' % file_list)
return expression, file_list
def shape_list(expression=None, file_list=None, axes_order=None):
"""Calculates the shape of the data in a file list.
Arguments
---------
expression : str or None
The regular epression for the file list.
file_list : list or None
List of files.
axes_order : list or None
The names of how to order the different tag names in the expression.
If None, use ordering of the tags in the expression.
Returns
-------
shape : int or tuple of ints
The shape of the array along the dimensions created by the file list.
"""
expression, file_list = _expression_and_file_list(expression=expression,
file_list=file_list)
if len(file_list) == 0:
raise ValueError(
'Cannot determine dimension of the file list %r without files.!' %
expression)
#ordering of the axes
if axes_order is None:
axes_order = expression.tag_names()
#determine dimensions along the file list
values0 = expression.values(file_list[0])
if len(axes_order) == 1:
shape_list = (len(file_list), )
else:
shape_list = ()
for a in axes_order:
values = values0.copy()
values.__delitem__(a)
e = te.Expression(expression.string(values=values))
search = re.compile(e.re()).search
shape_list += (len([f for f in file_list if search(f)]), )
return shape_list
def _expression_or_file_list(expression=None, file_list=None):
if isinstance(expression, te.Expression) or expression is None:
pass
elif fu.is_directory(expression):
file_list = _file_list(expression=expression, sort=True)
expression = None
elif isinstance(expression, str):
expression = te.Expression(expression)
else:
raise ValueError('The expression %r is not valid!' % expression)
if file_list is not None and not isinstance(file_list, list):
raise ValueError('The file_list %r is not a list or None!' % file_list)
if expression is None and file_list is None:
raise ValueError('Expresson and file_list cannot both be None!')
return expression, file_list
def _file_list(expression=None, file_list=None, sort=True, verbose=False):
"""Returns the list of files that match the tag expression.
Arguments
---------
expression :str
The regular expression the file names should match.
sort : bool
If True, sort files naturally.
verbose : bool
If True, print warning if no files exists.
Returns
-------
file_list : list of str
The list of files that matched the expression.
"""
if isinstance(file_list, list):
return file_list
if isinstance(expression, te.Expression):
fl = glob.glob(expression.glob())
elif fu.is_directory(expression):
expression = fu.join(expression, '*')
fl = glob.glob(expression)
else:
e = te.Expression(expression)
fl = glob.glob(e.glob())
if verbose and len(fl) == 0:
warnings.warn('No files found matching %s !' % expression)
return []
if sort:
fl = natsort.natsorted(fl)
return fl
def filename(ftype, file_type_to_name = None, directory = None, expression = None, values = None, prefix = None, postfix = None, extension = None, debug = None):
"""Returns the standard file name to use for a result file.
Arguments
---------
ftype : str
The type of the file for which the file name is requested.
directory : str
The working directory of the project.
expression : str or None
The tag expression to use if ftype is 'expression'.
file_type_to_name : dict
The file types to name mappings. If None, the default is used.
values : dict or None
The values to use in case a tag expression is given.
prefix : str or None
Optional prefix to the file if not None.
postfix : str or list of str or None
Optional postfix to the file if not None.
extension : str or None
Optional extension to replace existing one.
debug : str, bool or None
Optional string for debug files in wihch the string is added as postfix.
If True, 'debug' is added.
Returns
-------
filename : str
The standard file name of the requested file type.
"""
if file_type_to_name is None:
file_type_to_name = default_file_type_to_name;
if ftype is None:
fname = directory;
if ftype in file_type_synonyms.keys():
ftype = file_type_synonyms[ftype];
if ftype == 'expression' or expression is not None:
expression = te.Expression(expression);
fname = expression.string(values=values);
#Note: expressions are used for raw data only atm -> no prefix, debug
prefix = None;
debug = None;
else:
fname = file_type_to_name.get(ftype, None);
e = te.Expression(fname);
if len(e.tags) > 0:
fname = e.string(values=values);
#Note: expressions are used for raw data only atm -> no prefix, debug
prefix = None;
debug = None;
if fname is None:
raise ValueError('Cannot find name for type %r!' % ftype);
if prefix and prefix != '':
if isinstance(prefix, list):
prefix = '_'.join(prefix);
fname = prefix + '_' + fname;
if postfix is not None and postfix != '':
if isinstance(postfix, list):
postfix = '_'.join(postfix);
fname = fname.split('.');
fname = '.'.join(fname[:-1]) + '_' + postfix + '.' + fname[-1];
if debug:
if not isinstance(debug, str):
debug = 'debug';
#fname = fname.split('.');
#fname = '.'.join(fname[:-1]) + '_' + debug + '.' + fname[-1];
fname = debug + '_' + fname;
if extension:
fname = fname.split('.');
fname = '.'.join(fname[:-1] + [extension]);
if directory:
fname = io.join(directory, fname);
return fname;
def info(self, tile_axes = None, check_extensions = True):
s = self.__str__() + '\n';
l = np.max([len(k) for k in self.file_type_to_name]);
l = '%' + '%d' % l + 's';
for k,v in self.file_type_to_name.items():
if len(te.Expression(v).tags) > 0:
if check_extensions:
files = self.file_list(k, extension='*');
extensions = [io.file_extension(f) for f in files];
extensions = np.unique(extensions);
#print(extensions)
else:
extensions = [self.extension(k)];
if len(extensions) == 0:
s += l % k + ': no file\n';
else:
kk = k;
for extension in extensions:
expression = te.Expression(self.filename(k, extension=extension));
tag_names = expression.tag_names();
if tile_axes is None:
tile_axes_ = tag_names;
else:
tile_axes_ = tile_axes;
for n in tile_axes_:
if not n in tag_names:
raise ValueError('The expression does not have the named pattern %s' % n);
for n in tag_names:
if not n in tile_axes_:
raise ValueError('The expression has the named pattern %s that is not in tile_axes=%r' % (n, tile_axes_));
#construct tiling
files = io.file_list(expression);
if len(files) > 0:
tile_positions = [expression.values(f) for f in files];
tile_positions = [tuple(tv[n] for n in tile_axes_) for tv in tile_positions];
tile_lower = tuple(np.min(tile_positions, axis = 0));
tile_upper = tuple(np.max(tile_positions, axis = 0));
tag_names = tuple(tag_names);
if kk is not None:
s += (l % kk) + ': '
kk = None;
else:
s += (l % '') + ' '
s+= ('%s {%d files, %r: %r -> %r}' % (expression.string()[len(self.directory)+1:], len(files), tag_names, tile_lower, tile_upper)) + '\n';
else:
fname = self.filename(k);
files = [];
if io.is_file(fname):
files += [fname];
fname = self.filename(k, postfix = '*');
files += io.file_list(fname);
if len(files) > 0:
files = [f[len(self.directory)+1:] for f in files]
s += l % k + ': ' + files[0] + '\n'
for f in files[1:]:
s += l % '' + ' ' + f + '\n'
else:
s += l % k + ': no file\n';
print(s);
def expression(self, *args,**kwargs): return te.Expression(self.filename(*args, **kwargs));
def is_file_list(expression,
exists=False,
tag_names=None,
n_tags=-1,
verbose=False):
"""Checks if the expression is a valid file list.
Arguments
---------
expression : str
The regular expression to check.
exists : bool
If True, check if at least one file exists.
tag_names : list of str or None
List of tag names expected to be present in the expression.
n_Tags : int or None
Number of tags to expect.
verbose : bool
If True, print reason why the epxression does not represent the desired file list.
Returns
-------
is_expression : bool
Returns True if the expression fullfills the desired criteria and at least one file matching the expression exists.
"""
if isinstance(expression, Source):
return True
if not isinstance(expression, (str, te.Expression)):
if verbose:
warnings.warn(
'The expression %r is not a string or valid Source!' %
expression)
return False
if fu.is_directory(expression):
if exists:
if len(os.listdir(expression)) == 0:
if verbose:
warnings.warn('No files exists in the directory %s!' %
expression)
return False
else:
return True
else:
return True
if tag_names is not None or n_tags is not None:
t = te.Expression(expression) if not isinstance(
expression, te.Expression) else expression
if n_tags is not None:
if n_tags < 0 and -n_tags > t.ntags():
if verbose:
warnings.warn(
'Expression has not required number %d of tags, but %d!'
% (n_tags, t.ntags()))
return False
elif n_tags >= 0 and n_tags != t.ntags():
if verbose:
warnings.warn(
'Expression has not required number %d of tags, but %d!'
% (n_tags, t.ntags()))
return False
if tag_names is not None:
if tag_names != t.tag_names():
if verbose:
warnings.warn(
'Expression has not required tags %r, but %r!' %
(tag_names, t.tag_names()))
return False
if exists:
f = _first_file(expression)
if f is None:
if verbose:
warnings.warn('Expression does not point to any files!')
return False
return True