def finalized(self): mapreduce_id = self.outputs.job_id.value mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id) params = mapreduce_state.mapreduce_spec.mapper.params finalized_func = params.get('_finalized', None) if not finalized_func: return None finalized_func = for_name(finalized_func) return finalized_func(job_name=self.args[0], outputs=self.outputs, *params.get('args', []), **params.get('kwargs', {}))
def callback(self, **kwargs): """ Callback finish exists on the pipeline class, so we just use it as a nice wrapper for the static method attached to the MapReduceTask """ ctx = context.get() params = ctx.mapreduce_spec.mapper.params finish_func = params.get('_finish', None) if not finish_func: return None finish_func = for_name(finish_func) return finish_func(**kwargs)
def thunk_map(x): """ This is the default map function that wraps the static map function. It allows you to pass args and kwargs to your map function for defining more dynamic mappers """ ctx = context.get() params = ctx.mapreduce_spec.mapper.params map_func = params.get('_map', None) args = params.get('args', []) kwargs = params.get('kwargs', {}) map_func = for_name(map_func) return map_func(x, *args, **kwargs)
def callback(self): """ Callback finish exists on the pipeline class, so we just use it as a nice wrapper for the static method attached to the MapReduceTask """ super(DjangaeMapperPipeline, self).callback() mapreduce_id = self.outputs.job_id.value mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id) params = mapreduce_state.mapreduce_spec.mapper.params finish_func = params.get('_finish', None) if not finish_func: return None finish_func = for_name(finish_func) return finish_func(*params.get('args',[]), **params.get('kwargs', {}))
def callback(self): """ Callback finish exists on the pipeline class, so we just use it as a nice wrapper for the static method attached to the MapReduceTask """ super(DjangaeMapperPipeline, self).callback() mapreduce_id = self.outputs.job_id.value mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id) params = mapreduce_state.mapreduce_spec.mapper.params finish_func = params.get('_finish', None) if not finish_func: return None finish_func = for_name(finish_func) return finish_func(*params.get('args', []), **params.get('kwargs', {}))