def gen(count = None, start = 0, pad = None): """Generate a sequence of C{count} integers, starting at C{start}. If no arguments are provided then the sequence starts at 0 and does not terminate. If C{pad} is specified, and is an integer, then the output values are strings, padded to the specified length. The length of the generated strings must not exceed C{pad}. """ args = [count, start] if pad is not None: args.append(Option('-p', pad)) return _Gen().process_args(*args)
def unique(consecutive = False): """Input objects are passed to output, removing duplicates. No output is generated until the end of the input stream occurs. However, if the duplicates are known to be consecutive, then setting C{consecutive} to true allows output to be generated sooner. Input order is preserved only if C{consecutive} is true. """ args = [] if consecutive: args.append(Option('-c')) return _Unique().process_args(*args)
def unique(consecutive=False): """Input objects are passed to output, removing duplicates. No output is generated until the end of the input stream occurs. However, if the duplicates are known to be consecutive, then setting C{consecutive} to true allows output to be generated sooner. Input order is preserved only if C{consecutive} is true. """ args = [] if consecutive: args.append(Option('-c')) return _Unique().process_args(*args)
def red(binary_functions, running=False): """C{binary_functions} is a sequence. Each element of C{binary_functions} is either None or a binary function. If no elements are C{None}, then the binary function in position C{i} is used to reduce the values in element C{i} of the input sequences. If there are C{None} values, then these are used to define groups of inputs, partitioning by the values in the indicated columns. The remaining binary functions then compute reductions within each group. If C{running} is C{False} then there is one output tuple in the absence of grouping; otherwise there is one tuple output per group. If C{running} is {True}, then the aggregate value computed so far is written out for each input, i.e., the output contains "running totals". In this case, the aggregate value appears before the input values. """ if not (isinstance(binary_functions, list) or isinstance(binary_functions, tuple)): binary_functions = [binary_functions] args = [f for f in binary_functions] if running: args.append(Option('-r')) return _Red().process_args(*args)
def red(binary_functions, running = False): """C{binary_functions} is a sequence. Each element of C{binary_functions} is either None or a binary function. If no elements are C{None}, then the binary function in position C{i} is used to reduce the values in element C{i} of the input sequences. If there are C{None} values, then these are used to define groups of inputs, partitioning by the values in the indicated columns. The remaining binary functions then compute reductions within each group. If C{running} is C{False} then there is one output tuple in the absence of grouping; otherwise there is one tuple output per group. If C{running} is {True}, then the aggregate value computed so far is written out for each input, i.e., the output contains "running totals". In this case, the aggregate value appears before the input values. """ if not (isinstance(binary_functions, list) or isinstance(binary_functions, tuple)): binary_functions = [binary_functions] args = [f for f in binary_functions] if running: args.append(Option('-r')) return _Red().process_args(*args)
def out(file=None, append=None, format=None, csv=False, terminal=False): """Prints input objects. Each input object is rendered as a string, printed, and passed on as output (unless C{terminal} is true). The format of the object is the default (obtained by applying C{str}), determined by C{format} if specified. The I{comma-separated values} format is used if C{csv} is true. Output is written to C{stdout} unless a filename is specified using C{append} or C{file}. C{append} causes output to be appended to the named file, while C{file} causes the file to be created or replaced. """ args = [] if append: args.append(Option('-a', append)) if file: args.append(Option('-f', file)) if csv: args.append(Option('-c')) if terminal: args.append(Option('-t')) if format: args.append(format) return _Out().process_args(*args)
def out(file = None, append = None, format = None, csv = False, terminal = False): """Prints input objects. Each input object is rendered as a string, printed, and passed on as output (unless C{terminal} is true). The format of the object is the default (obtained by applying C{str}), determined by C{format} if specified. The I{comma-separated values} format is used if C{csv} is true. Output is written to C{stdout} unless a filename is specified using C{append} or C{file}. C{append} causes output to be appended to the named file, while C{file} causes the file to be created or replaced. """ args = [] if append: args.append(Option('-a', append)) if file: args.append(Option('-f', file)) if csv: args.append(Option('-c')) if terminal: args.append(Option('-t')) if format: args.append(format) return _Out().process_args(*args)
def agg(initial_value, aggregator, group = None, consecutive = None, running = False): """Combine inputs into a smaller number of outputs. If neither C{group} nor C{consecutive} is specified, then there is one accumulator, initialized to C{initial_value}. The C{aggregator} function is used to combine the current value of the accumulator with the input to yield the next value of the accumulator. The arguments to C{aggregator} are the elements of the accumulator followed by the elements of one piece of input. If C{group} is specified, then there is one accumulator for each group value, defined by applying the function C{group} to each input. C{consecutive} is just like C{group} except that it is assumed that group values are adjacent in the input sequence. At most one of C{group} and C{consecutive} may be specified. If C{running} is C{false}, then output contains one object per group, containing the aggregate value. (If neither C{group} nor C{consecutive} are provided, then there is just one group, representing the aggregate for the entire input stream.) If C{running} is true, then each the aggregate value for the group is written out with each input object -- i.e., the output contains "running totals". In this case, the aggregate values appear before the input values in the output object. """ args = [initial_value, aggregator] if group: args.append(Option('-g', group)) if consecutive: args.append(Option('-c', consecutive)) if running: args.append(Option('-r')) return _Agg().process_args(*args)
def agg(initial_value, aggregator, group=None, consecutive=None, running=False): """Combine inputs into a smaller number of outputs. If neither C{group} nor C{consecutive} is specified, then there is one accumulator, initialized to C{initial_value}. The C{aggregator} function is used to combine the current value of the accumulator with the input to yield the next value of the accumulator. The arguments to C{aggregator} are the elements of the accumulator followed by the elements of one piece of input. If C{group} is specified, then there is one accumulator for each group value, defined by applying the function C{group} to each input. C{consecutive} is just like C{group} except that it is assumed that group values are adjacent in the input sequence. At most one of C{group} and C{consecutive} may be specified. If C{running} is C{false}, then output contains one object per group, containing the aggregate value. (If neither C{group} nor C{consecutive} are provided, then there is just one group, representing the aggregate for the entire input stream.) If C{running} is true, then each the aggregate value for the group is written out with each input object -- i.e., the output contains "running totals". In this case, the aggregate values appear before the input values in the output object. """ args = [initial_value, aggregator] if group: args.append(Option('-g', group)) if consecutive: args.append(Option('-c', consecutive)) if running: args.append(Option('-r')) return _Agg().process_args(*args)
def copyfrom(files, local_dir, compress=False, recursive=False, preserve=False, no_subdirs=False): """Copies C{files} from each node of the specified C{cluster} to C{local_dir}. If C{no_subdirs} is False, then a subdirectory under C{local_dir} is created for each node of the cluster and the files from a node are copied to that node's subdirectory. If C{no_subdirs} is true, then C{cluster} must be a single-node cluster, no subdirectory is created, and files are copied directly into C{local_dir}. Compression is used for copying if C{compress} is True. Directories are copied recursively if C{recursive} is True. File attributes are preserved if C{preserve} is True. """ args = [] if compress: args.append(Option("-C")) if recursive: args.append(Option("-r")) if preserve: args.append(Option("-p")) if no_subdirs: args.append(Option("-x")) return _CopyFrom().process_args(*args)
def ls(paths, depth_0 = False, depth_1 = False, recursive = False, file = False, dir = False, link = False): """Generates a stream of objects of type C{osh.file.File}. C{paths} may be a single path or a sequence of paths Each path in C{paths} is expanded as a Unix "glob" pattern. If no path is specified, then the files in the current directory are output. If C{depth_0} is True, then only files matching the specified pathss are output; any qualifying directories are not explored. If C{depth_1} is True, then behavior is as for C{depth_0} except that files in qualifying directories are output as well. If C{recursive} is True, then directories are recursively explored. At most one of the last three arguments may be True. If all are False, then C{depth_1} behavior is obtained. Files are listed if and only if C{file} is true. Directories are listed if and only if C{dir} is true. Symlinks are listed if and only if C{link} is true. If C{file}, C{dir} and C{link} are all false, then files, directories and symlinks are listed. Symlinks to directories are not explored. """ args = [] if depth_0: args.append(Option('-0')) if depth_1: args.append(Option('-1')) if recursive: args.append(Option('-r')) if file: args.append(Option('-f')) if dir: args.append(Option('-d')) if link: args.append(Option('-s')) if isinstance(paths, str): paths = [paths] args.extend(paths) return _Ls().process_args(*args)