def expression(self): """The underlying expression of this file list. Returns ------- expression : str The underlying expression of this source. """ if self._expression is None: self._expression = te.detect(self.file_list) return self._expression
def _expression_and_file_list(expression=None, file_list=None): if isinstance(expression, te.Expression) or expression is None: pass elif fu.is_directory(expression): if file_list is None: file_list = glob.glob(fu.join(expression, '*')) expression = te.detect(file_list) elif isinstance(expression, (str, list)): expression = te.Expression(expression) else: raise ValueError('Expression %r is not valid!' % expression) if file_list is None: if expression is None: raise ValueError( 'Either expression or file_list need to be specified!') file_list = glob.glob(expression.glob()) elif isinstance(file_list, list): if expression is None: expression = te.detect(file_list) else: raise ValueError('The file_list %r is not value!' % file_list) return expression, file_list
def shape_list(expression=None, file_list=None, axes_order=None): """Calculates the shape of the data in a file list. Arguments --------- expression : str or None The regular epression for the file list. file_list : list or None List of files. axes_order : list or None The names of how to order the different tag names in the expression. If None, use ordering of the tags in the expression. Returns ------- shape : int or tuple of ints The shape of the array along the dimensions created by the file list. """ expression, file_list = _expression_and_file_list(expression=expression, file_list=file_list) if len(file_list) == 0: raise ValueError( 'Cannot determine dimension of the file list %r without files.!' % expression) #ordering of the axes if axes_order is None: axes_order = expression.tag_names() #determine dimensions along the file list values0 = expression.values(file_list[0]) if len(axes_order) == 1: shape_list = (len(file_list), ) else: shape_list = () for a in axes_order: values = values0.copy() values.__delitem__(a) e = te.Expression(expression.string(values=values)) search = re.compile(e.re()).search shape_list += (len([f for f in file_list if search(f)]), ) return shape_list
def _expression_or_file_list(expression=None, file_list=None): if isinstance(expression, te.Expression) or expression is None: pass elif fu.is_directory(expression): file_list = _file_list(expression=expression, sort=True) expression = None elif isinstance(expression, str): expression = te.Expression(expression) else: raise ValueError('The expression %r is not valid!' % expression) if file_list is not None and not isinstance(file_list, list): raise ValueError('The file_list %r is not a list or None!' % file_list) if expression is None and file_list is None: raise ValueError('Expresson and file_list cannot both be None!') return expression, file_list
def _file_list(expression=None, file_list=None, sort=True, verbose=False): """Returns the list of files that match the tag expression. Arguments --------- expression :str The regular expression the file names should match. sort : bool If True, sort files naturally. verbose : bool If True, print warning if no files exists. Returns ------- file_list : list of str The list of files that matched the expression. """ if isinstance(file_list, list): return file_list if isinstance(expression, te.Expression): fl = glob.glob(expression.glob()) elif fu.is_directory(expression): expression = fu.join(expression, '*') fl = glob.glob(expression) else: e = te.Expression(expression) fl = glob.glob(e.glob()) if verbose and len(fl) == 0: warnings.warn('No files found matching %s !' % expression) return [] if sort: fl = natsort.natsorted(fl) return fl
def filename(ftype, file_type_to_name = None, directory = None, expression = None, values = None, prefix = None, postfix = None, extension = None, debug = None): """Returns the standard file name to use for a result file. Arguments --------- ftype : str The type of the file for which the file name is requested. directory : str The working directory of the project. expression : str or None The tag expression to use if ftype is 'expression'. file_type_to_name : dict The file types to name mappings. If None, the default is used. values : dict or None The values to use in case a tag expression is given. prefix : str or None Optional prefix to the file if not None. postfix : str or list of str or None Optional postfix to the file if not None. extension : str or None Optional extension to replace existing one. debug : str, bool or None Optional string for debug files in wihch the string is added as postfix. If True, 'debug' is added. Returns ------- filename : str The standard file name of the requested file type. """ if file_type_to_name is None: file_type_to_name = default_file_type_to_name; if ftype is None: fname = directory; if ftype in file_type_synonyms.keys(): ftype = file_type_synonyms[ftype]; if ftype == 'expression' or expression is not None: expression = te.Expression(expression); fname = expression.string(values=values); #Note: expressions are used for raw data only atm -> no prefix, debug prefix = None; debug = None; else: fname = file_type_to_name.get(ftype, None); e = te.Expression(fname); if len(e.tags) > 0: fname = e.string(values=values); #Note: expressions are used for raw data only atm -> no prefix, debug prefix = None; debug = None; if fname is None: raise ValueError('Cannot find name for type %r!' % ftype); if prefix and prefix != '': if isinstance(prefix, list): prefix = '_'.join(prefix); fname = prefix + '_' + fname; if postfix is not None and postfix != '': if isinstance(postfix, list): postfix = '_'.join(postfix); fname = fname.split('.'); fname = '.'.join(fname[:-1]) + '_' + postfix + '.' + fname[-1]; if debug: if not isinstance(debug, str): debug = 'debug'; #fname = fname.split('.'); #fname = '.'.join(fname[:-1]) + '_' + debug + '.' + fname[-1]; fname = debug + '_' + fname; if extension: fname = fname.split('.'); fname = '.'.join(fname[:-1] + [extension]); if directory: fname = io.join(directory, fname); return fname;
def info(self, tile_axes = None, check_extensions = True): s = self.__str__() + '\n'; l = np.max([len(k) for k in self.file_type_to_name]); l = '%' + '%d' % l + 's'; for k,v in self.file_type_to_name.items(): if len(te.Expression(v).tags) > 0: if check_extensions: files = self.file_list(k, extension='*'); extensions = [io.file_extension(f) for f in files]; extensions = np.unique(extensions); #print(extensions) else: extensions = [self.extension(k)]; if len(extensions) == 0: s += l % k + ': no file\n'; else: kk = k; for extension in extensions: expression = te.Expression(self.filename(k, extension=extension)); tag_names = expression.tag_names(); if tile_axes is None: tile_axes_ = tag_names; else: tile_axes_ = tile_axes; for n in tile_axes_: if not n in tag_names: raise ValueError('The expression does not have the named pattern %s' % n); for n in tag_names: if not n in tile_axes_: raise ValueError('The expression has the named pattern %s that is not in tile_axes=%r' % (n, tile_axes_)); #construct tiling files = io.file_list(expression); if len(files) > 0: tile_positions = [expression.values(f) for f in files]; tile_positions = [tuple(tv[n] for n in tile_axes_) for tv in tile_positions]; tile_lower = tuple(np.min(tile_positions, axis = 0)); tile_upper = tuple(np.max(tile_positions, axis = 0)); tag_names = tuple(tag_names); if kk is not None: s += (l % kk) + ': ' kk = None; else: s += (l % '') + ' ' s+= ('%s {%d files, %r: %r -> %r}' % (expression.string()[len(self.directory)+1:], len(files), tag_names, tile_lower, tile_upper)) + '\n'; else: fname = self.filename(k); files = []; if io.is_file(fname): files += [fname]; fname = self.filename(k, postfix = '*'); files += io.file_list(fname); if len(files) > 0: files = [f[len(self.directory)+1:] for f in files] s += l % k + ': ' + files[0] + '\n' for f in files[1:]: s += l % '' + ' ' + f + '\n' else: s += l % k + ': no file\n'; print(s);
def expression(self, *args,**kwargs): return te.Expression(self.filename(*args, **kwargs));
def is_file_list(expression, exists=False, tag_names=None, n_tags=-1, verbose=False): """Checks if the expression is a valid file list. Arguments --------- expression : str The regular expression to check. exists : bool If True, check if at least one file exists. tag_names : list of str or None List of tag names expected to be present in the expression. n_Tags : int or None Number of tags to expect. verbose : bool If True, print reason why the epxression does not represent the desired file list. Returns ------- is_expression : bool Returns True if the expression fullfills the desired criteria and at least one file matching the expression exists. """ if isinstance(expression, Source): return True if not isinstance(expression, (str, te.Expression)): if verbose: warnings.warn( 'The expression %r is not a string or valid Source!' % expression) return False if fu.is_directory(expression): if exists: if len(os.listdir(expression)) == 0: if verbose: warnings.warn('No files exists in the directory %s!' % expression) return False else: return True else: return True if tag_names is not None or n_tags is not None: t = te.Expression(expression) if not isinstance( expression, te.Expression) else expression if n_tags is not None: if n_tags < 0 and -n_tags > t.ntags(): if verbose: warnings.warn( 'Expression has not required number %d of tags, but %d!' % (n_tags, t.ntags())) return False elif n_tags >= 0 and n_tags != t.ntags(): if verbose: warnings.warn( 'Expression has not required number %d of tags, but %d!' % (n_tags, t.ntags())) return False if tag_names is not None: if tag_names != t.tag_names(): if verbose: warnings.warn( 'Expression has not required tags %r, but %r!' % (tag_names, t.tag_names())) return False if exists: f = _first_file(expression) if f is None: if verbose: warnings.warn('Expression does not point to any files!') return False return True