class BSONReducer(object):
def __init__(self, factory=None, input_fh=None):
if factory:
# Try to figure out if they gave us a factory
# or a straight function
spec = inspect.getargspec(factory)
if len(spec.args) == 2:
self.factory = lambda: factory
elif len(spec.args) == 0:
self.factory = factory
else:
raise ValueError("Invalid Factory. Must return a function expecting 2 arguments or be a function expecting 2 arguments.")
if not input_fh:
self.input = BSONReducerInput(self)
else:
self.input = input_fh
self.output = BSONOutput()
self.output.writes(self.input)
def __call__(self, data):
for key, values in data:
yield self.factory()(key, values)
def factory(self):
"""Processor factory used to consume reducer input
Must return a callable (aka processor) that accepts two parameters
"key" and "values", and returns an iterable of strings or None.
"""
return lambda key, values: {'values': [v for v in values]}
class BSONReducer(object):
output = BSONOutput()
def __init__(self, factory=None, input_fh=None):
if factory:
# Try to figure out if they gave us a factory
# or a straight function
spec = inspect.getargspec(factory)
if len(spec.args) == 2:
self.factory = lambda: factory
elif len(spec.args) == 0:
self.factory = factory
else:
raise ValueError(
"Invalid Factory. Must return a function expecting 2 arguments or be a function expecting 2 arguments."
)
if not input_fh:
self.input = BSONReducerInput(self)
else:
self.input = input_fh
self.output.writes(self.input)
def __call__(self, data):
for key, values in data:
yield self.factory()(key, values)
def factory(self):
"""Processor factory used to consume reducer input
Must return a callable (aka processor) that accepts two parameters
"key" and "values", and returns an iterable of strings or None.
"""
return lambda key, values: {'values': [v for v in values]}
def __init__(self, target, **kwargs):
"""`target` should be a generator function that accepts a
single argument which will be an instance of :class:`BSONInput`,
and which yields dictionaries to be emitted. The yielded
dictionaries should conform to the format expected by
:class:`BSONInput` (i.e. they should have the key defined
in a field named `_id`).
Keyword arguments are passed directly to the underlying
:class:`BSONInput`.
"""
output = BSONOutput()
input = BSONInput(**kwargs)
generator = target(input)
for mapped in generator:
output.write(mapped)
def __init__(self, target, **kwargs):
"""`target` should be a generator function that accepts a
single argument which will be an instance of :class:`BSONInput`,
and which yields dictionaries to be emitted. The yielded
dictionaries should conform to the format expected by
:class:`BSONInput` (i.e. they should have the key defined
in a field named `_id`).
Keyword arguments are passed directly to the underlying
:class:`BSONInput`.
"""
output = BSONOutput()
input = BSONInput(**kwargs)
generator = target(input)
for mapped in generator:
output.write(mapped)
def __init__(self, factory=None, input_fh=None):
if factory:
# Try to figure out if they gave us a factory
# or a straight function
spec = inspect.getargspec(factory)
if len(spec.args) == 2:
self.factory = lambda: factory
elif len(spec.args) == 0:
self.factory = factory
else:
raise ValueError("Invalid Factory. Must return a function expecting 2 arguments or be a function expecting 2 arguments.")
if not input_fh:
self.input = BSONReducerInput(self)
else:
self.input = input_fh
self.output = BSONOutput()
self.output.writes(self.input)