class BSONReducer(object):
    def __init__(self, factory=None, input_fh=None):
        if factory:
            # Try to figure out if they gave us a factory
            # or a straight function
            spec = inspect.getargspec(factory)
            if len(spec.args) == 2:
                self.factory = lambda: factory
            elif len(spec.args) == 0:
                self.factory = factory
            else:
                raise ValueError("Invalid Factory.  Must return a function expecting 2 arguments or be a function expecting 2 arguments.")
            
        if not input_fh:
            self.input = BSONReducerInput(self)
        else:
            self.input = input_fh

        self.output = BSONOutput()

        self.output.writes(self.input)

    def __call__(self, data):
        for key, values in data:
            yield self.factory()(key, values) 

    def factory(self):
        """Processor factory used to consume reducer input        
        Must return a callable (aka processor) that accepts two parameters
        "key" and "values", and returns an iterable of strings or None.
        """
        return lambda key, values: {'values': [v for v in values]}
Exemple #2
0
class BSONReducer(object):

    output = BSONOutput()

    def __init__(self, factory=None, input_fh=None):
        if factory:
            # Try to figure out if they gave us a factory
            # or a straight function
            spec = inspect.getargspec(factory)
            if len(spec.args) == 2:
                self.factory = lambda: factory
            elif len(spec.args) == 0:
                self.factory = factory
            else:
                raise ValueError(
                    "Invalid Factory.  Must return a function expecting 2 arguments or be a function expecting 2 arguments."
                )

        if not input_fh:
            self.input = BSONReducerInput(self)
        else:
            self.input = input_fh

        self.output.writes(self.input)

    def __call__(self, data):
        for key, values in data:
            yield self.factory()(key, values)

    def factory(self):
        """Processor factory used to consume reducer input
        Must return a callable (aka processor) that accepts two parameters
        "key" and "values", and returns an iterable of strings or None.
        """
        return lambda key, values: {'values': [v for v in values]}
Exemple #3
0
    def __init__(self, target, **kwargs):
        """`target` should be a generator function that accepts a
        single argument which will be an instance of :class:`BSONInput`,
        and which yields dictionaries to be emitted. The yielded
        dictionaries should conform to the format expected by
        :class:`BSONInput` (i.e. they should have the key defined
        in a field named `_id`).

        Keyword arguments are passed directly to the underlying
        :class:`BSONInput`.
        """

        output = BSONOutput()
        input = BSONInput(**kwargs)

        generator = target(input)
        for mapped in generator:
            output.write(mapped)
Exemple #4
0
    def __init__(self, target, **kwargs):
        """`target` should be a generator function that accepts a
        single argument which will be an instance of :class:`BSONInput`,
        and which yields dictionaries to be emitted. The yielded
        dictionaries should conform to the format expected by
        :class:`BSONInput` (i.e. they should have the key defined
        in a field named `_id`).

        Keyword arguments are passed directly to the underlying
        :class:`BSONInput`.
        """

        output = BSONOutput()
        input = BSONInput(**kwargs)

        generator = target(input)
        for mapped in generator:
            output.write(mapped)
    def __init__(self, factory=None, input_fh=None):
        if factory:
            # Try to figure out if they gave us a factory
            # or a straight function
            spec = inspect.getargspec(factory)
            if len(spec.args) == 2:
                self.factory = lambda: factory
            elif len(spec.args) == 0:
                self.factory = factory
            else:
                raise ValueError("Invalid Factory.  Must return a function expecting 2 arguments or be a function expecting 2 arguments.")
            
        if not input_fh:
            self.input = BSONReducerInput(self)
        else:
            self.input = input_fh

        self.output = BSONOutput()

        self.output.writes(self.input)