def gen(count = None, start = 0, pad = None):
"""Generate a sequence of C{count} integers, starting at C{start}. If no arguments
are provided then the sequence starts at 0 and does not terminate. If C{pad} is
specified, and is an integer, then the output values are strings, padded to the
specified length. The length of the generated strings must not exceed C{pad}.
"""
args = [count, start]
if pad is not None:
args.append(Option('-p', pad))
return _Gen().process_args(*args)
def unique(consecutive = False):
"""Input objects are passed to output, removing duplicates. No output
is generated until the end of the input stream occurs. However, if the
duplicates are known to be consecutive, then setting C{consecutive} to true
allows output to be generated sooner. Input order is preserved only if
C{consecutive} is true.
"""
args = []
if consecutive:
args.append(Option('-c'))
return _Unique().process_args(*args)
def unique(consecutive=False):
"""Input objects are passed to output, removing duplicates. No output
is generated until the end of the input stream occurs. However, if the
duplicates are known to be consecutive, then setting C{consecutive} to true
allows output to be generated sooner. Input order is preserved only if
C{consecutive} is true.
"""
args = []
if consecutive:
args.append(Option('-c'))
return _Unique().process_args(*args)
def red(binary_functions, running=False):
"""C{binary_functions} is a sequence. Each element of C{binary_functions} is
either None or a binary function.
If no elements are C{None}, then the binary function in position C{i} is used
to reduce the values in element C{i} of the input sequences. If there are
C{None} values, then these are used to define groups of inputs, partitioning
by the values in the indicated columns. The remaining binary functions then compute
reductions within each group. If C{running} is C{False} then there is one output
tuple in the absence of grouping; otherwise there is one tuple output per group.
If C{running} is {True}, then the aggregate value computed so far is written
out for each input, i.e., the output contains "running totals". In this case,
the aggregate value appears before the input values.
"""
if not (isinstance(binary_functions, list)
or isinstance(binary_functions, tuple)):
binary_functions = [binary_functions]
args = [f for f in binary_functions]
if running:
args.append(Option('-r'))
return _Red().process_args(*args)
def red(binary_functions, running = False):
"""C{binary_functions} is a sequence. Each element of C{binary_functions} is
either None or a binary function.
If no elements are C{None}, then the binary function in position C{i} is used
to reduce the values in element C{i} of the input sequences. If there are
C{None} values, then these are used to define groups of inputs, partitioning
by the values in the indicated columns. The remaining binary functions then compute
reductions within each group. If C{running} is C{False} then there is one output
tuple in the absence of grouping; otherwise there is one tuple output per group.
If C{running} is {True}, then the aggregate value computed so far is written
out for each input, i.e., the output contains "running totals". In this case,
the aggregate value appears before the input values.
"""
if not (isinstance(binary_functions, list) or
isinstance(binary_functions, tuple)):
binary_functions = [binary_functions]
args = [f for f in binary_functions]
if running:
args.append(Option('-r'))
return _Red().process_args(*args)
def out(file=None, append=None, format=None, csv=False, terminal=False):
"""Prints input objects. Each input object is rendered as a string, printed,
and passed on as output (unless C{terminal} is true). The format of the object
is the default (obtained by applying C{str}), determined by C{format} if specified.
The I{comma-separated values} format is used if C{csv} is true. Output is written
to C{stdout} unless a filename is specified using C{append} or C{file}. C{append}
causes output to be appended to the named file, while C{file} causes the file to be
created or replaced.
"""
args = []
if append:
args.append(Option('-a', append))
if file:
args.append(Option('-f', file))
if csv:
args.append(Option('-c'))
if terminal:
args.append(Option('-t'))
if format:
args.append(format)
return _Out().process_args(*args)
def out(file = None, append = None, format = None, csv = False, terminal = False):
"""Prints input objects. Each input object is rendered as a string, printed,
and passed on as output (unless C{terminal} is true). The format of the object
is the default (obtained by applying C{str}), determined by C{format} if specified.
The I{comma-separated values} format is used if C{csv} is true. Output is written
to C{stdout} unless a filename is specified using C{append} or C{file}. C{append}
causes output to be appended to the named file, while C{file} causes the file to be
created or replaced.
"""
args = []
if append:
args.append(Option('-a', append))
if file:
args.append(Option('-f', file))
if csv:
args.append(Option('-c'))
if terminal:
args.append(Option('-t'))
if format:
args.append(format)
return _Out().process_args(*args)
def agg(initial_value,
aggregator,
group = None,
consecutive = None,
running = False):
"""Combine inputs into a smaller number of outputs. If neither C{group} nor
C{consecutive} is specified, then there is one accumulator, initialized to
C{initial_value}. The C{aggregator} function is used to combine the current value
of the accumulator with the input to yield the next value of the accumulator.
The arguments to C{aggregator} are the elements of the accumulator followed
by the elements of one piece of input.
If C{group} is specified, then there is one accumulator for each group value, defined
by applying the function C{group} to each input. C{consecutive} is just like C{group}
except that it is assumed that group values are adjacent in the input sequence.
At most one of C{group} and C{consecutive} may be specified. If C{running} is C{false},
then output contains one object per group, containing the aggregate value.
(If neither C{group} nor C{consecutive} are provided, then there is just one group,
representing the aggregate for the entire input stream.) If C{running} is true,
then each the aggregate value for the group is written out with each input object --
i.e., the output contains "running totals". In this case, the aggregate values appear
before the input values in the output object.
"""
args = [initial_value, aggregator]
if group:
args.append(Option('-g', group))
if consecutive:
args.append(Option('-c', consecutive))
if running:
args.append(Option('-r'))
return _Agg().process_args(*args)
def agg(initial_value,
aggregator,
group=None,
consecutive=None,
running=False):
"""Combine inputs into a smaller number of outputs. If neither C{group} nor
C{consecutive} is specified, then there is one accumulator, initialized to
C{initial_value}. The C{aggregator} function is used to combine the current value
of the accumulator with the input to yield the next value of the accumulator.
The arguments to C{aggregator} are the elements of the accumulator followed
by the elements of one piece of input.
If C{group} is specified, then there is one accumulator for each group value, defined
by applying the function C{group} to each input. C{consecutive} is just like C{group}
except that it is assumed that group values are adjacent in the input sequence.
At most one of C{group} and C{consecutive} may be specified. If C{running} is C{false},
then output contains one object per group, containing the aggregate value.
(If neither C{group} nor C{consecutive} are provided, then there is just one group,
representing the aggregate for the entire input stream.) If C{running} is true,
then each the aggregate value for the group is written out with each input object --
i.e., the output contains "running totals". In this case, the aggregate values appear
before the input values in the output object.
"""
args = [initial_value, aggregator]
if group:
args.append(Option('-g', group))
if consecutive:
args.append(Option('-c', consecutive))
if running:
args.append(Option('-r'))
return _Agg().process_args(*args)
def copyfrom(files, local_dir, compress=False, recursive=False, preserve=False, no_subdirs=False):
"""Copies C{files} from each node of the specified C{cluster} to C{local_dir}.
If C{no_subdirs} is False, then a subdirectory under C{local_dir} is created for
each node of the cluster and the files from a node are copied to that node's subdirectory.
If C{no_subdirs} is true, then C{cluster} must be a single-node cluster, no subdirectory
is created, and files are copied directly into C{local_dir}. Compression is used
for copying if C{compress} is True. Directories are copied recursively if C{recursive} is
True. File attributes are preserved if C{preserve} is True.
"""
args = []
if compress:
args.append(Option("-C"))
if recursive:
args.append(Option("-r"))
if preserve:
args.append(Option("-p"))
if no_subdirs:
args.append(Option("-x"))
return _CopyFrom().process_args(*args)
def ls(paths,
depth_0 = False,
depth_1 = False,
recursive = False,
file = False,
dir = False,
link = False):
"""Generates a stream of objects of type C{osh.file.File}. C{paths} may be a single
path or a sequence of paths
Each path in C{paths}
is expanded as a Unix "glob" pattern. If no path is specified,
then the files in the current directory are output.
If C{depth_0} is True, then only files matching the specified pathss are
output; any qualifying directories are not explored. If C{depth_1} is True,
then behavior is as for C{depth_0} except that files in qualifying directories
are output as well. If C{recursive} is True, then directories are recursively explored.
At most one of the last three arguments may be True. If all are False,
then C{depth_1} behavior is obtained. Files are listed if and only if C{file} is
true. Directories are listed if and only if C{dir} is true. Symlinks are listed
if and only if C{link} is true. If C{file}, C{dir} and C{link} are all false,
then files, directories and symlinks are listed. Symlinks to directories are
not explored.
"""
args = []
if depth_0:
args.append(Option('-0'))
if depth_1:
args.append(Option('-1'))
if recursive:
args.append(Option('-r'))
if file:
args.append(Option('-f'))
if dir:
args.append(Option('-d'))
if link:
args.append(Option('-s'))
if isinstance(paths, str):
paths = [paths]
args.extend(paths)
return _Ls().process_args(*args)
