class BSONReducer(object): def __init__(self, factory=None, input_fh=None): if factory: # Try to figure out if they gave us a factory # or a straight function spec = inspect.getargspec(factory) if len(spec.args) == 2: self.factory = lambda: factory elif len(spec.args) == 0: self.factory = factory else: raise ValueError("Invalid Factory. Must return a function expecting 2 arguments or be a function expecting 2 arguments.") if not input_fh: self.input = BSONReducerInput(self) else: self.input = input_fh self.output = BSONOutput() self.output.writes(self.input) def __call__(self, data): for key, values in data: yield self.factory()(key, values) def factory(self): """Processor factory used to consume reducer input Must return a callable (aka processor) that accepts two parameters "key" and "values", and returns an iterable of strings or None. """ return lambda key, values: {'values': [v for v in values]}
class BSONReducer(object): output = BSONOutput() def __init__(self, factory=None, input_fh=None): if factory: # Try to figure out if they gave us a factory # or a straight function spec = inspect.getargspec(factory) if len(spec.args) == 2: self.factory = lambda: factory elif len(spec.args) == 0: self.factory = factory else: raise ValueError( "Invalid Factory. Must return a function expecting 2 arguments or be a function expecting 2 arguments." ) if not input_fh: self.input = BSONReducerInput(self) else: self.input = input_fh self.output.writes(self.input) def __call__(self, data): for key, values in data: yield self.factory()(key, values) def factory(self): """Processor factory used to consume reducer input Must return a callable (aka processor) that accepts two parameters "key" and "values", and returns an iterable of strings or None. """ return lambda key, values: {'values': [v for v in values]}
def __init__(self, target, **kwargs): """`target` should be a generator function that accepts a single argument which will be an instance of :class:`BSONInput`, and which yields dictionaries to be emitted. The yielded dictionaries should conform to the format expected by :class:`BSONInput` (i.e. they should have the key defined in a field named `_id`). Keyword arguments are passed directly to the underlying :class:`BSONInput`. """ output = BSONOutput() input = BSONInput(**kwargs) generator = target(input) for mapped in generator: output.write(mapped)
def __init__(self, factory=None, input_fh=None): if factory: # Try to figure out if they gave us a factory # or a straight function spec = inspect.getargspec(factory) if len(spec.args) == 2: self.factory = lambda: factory elif len(spec.args) == 0: self.factory = factory else: raise ValueError("Invalid Factory. Must return a function expecting 2 arguments or be a function expecting 2 arguments.") if not input_fh: self.input = BSONReducerInput(self) else: self.input = input_fh self.output = BSONOutput() self.output.writes(self.input)