Beispiel #1
0
 def finalized(self):
     mapreduce_id = self.outputs.job_id.value
     mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
     params = mapreduce_state.mapreduce_spec.mapper.params
     finalized_func = params.get('_finalized', None)
     if not finalized_func:
         return None
     finalized_func = for_name(finalized_func)
     return finalized_func(job_name=self.args[0],
                           outputs=self.outputs,
                           *params.get('args', []),
                           **params.get('kwargs', {}))
Beispiel #2
0
 def callback(self, **kwargs):
     """
         Callback finish exists on the pipeline class, so we just use it as a nice
         wrapper for the static method attached to the MapReduceTask
     """
     ctx = context.get()
     params = ctx.mapreduce_spec.mapper.params
     finish_func = params.get('_finish', None)
     if not finish_func:
         return None
     finish_func = for_name(finish_func)
     return finish_func(**kwargs)
Beispiel #3
0
 def finalized(self):
     mapreduce_id = self.outputs.job_id.value
     mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
     params = mapreduce_state.mapreduce_spec.mapper.params
     finalized_func = params.get('_finalized', None)
     if not finalized_func:
         return None
     finalized_func = for_name(finalized_func)
     return finalized_func(job_name=self.args[0],
                           outputs=self.outputs,
                           *params.get('args', []),
                           **params.get('kwargs', {}))
def thunk_map(x):
    """
        This is the default map function that wraps the static map function.

        It allows you to pass args and kwargs to your map function for defining
        more dynamic mappers
    """
    ctx = context.get()
    params = ctx.mapreduce_spec.mapper.params
    map_func = params.get('_map', None)
    args = params.get('args', [])
    kwargs = params.get('kwargs', {})
    map_func = for_name(map_func)
    return map_func(x, *args, **kwargs)
Beispiel #5
0
    def callback(self):
        """
            Callback finish exists on the pipeline class, so we just use it as a nice
            wrapper for the static method attached to the MapReduceTask
        """
        super(DjangaeMapperPipeline, self).callback()

        mapreduce_id = self.outputs.job_id.value
        mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
        params = mapreduce_state.mapreduce_spec.mapper.params
        finish_func = params.get('_finish', None)
        if not finish_func:
            return None
        finish_func = for_name(finish_func)
        return finish_func(*params.get('args',[]), **params.get('kwargs', {}))
Beispiel #6
0
    def callback(self):
        """
            Callback finish exists on the pipeline class, so we just use it as a nice
            wrapper for the static method attached to the MapReduceTask
        """
        super(DjangaeMapperPipeline, self).callback()

        mapreduce_id = self.outputs.job_id.value
        mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
        params = mapreduce_state.mapreduce_spec.mapper.params
        finish_func = params.get('_finish', None)
        if not finish_func:
            return None
        finish_func = for_name(finish_func)
        return finish_func(*params.get('args', []), **params.get('kwargs', {}))