def get_status(pipeline_id): """Hack into the pipelines models to gather pipeline and mapreduce details.""" status_dict = { 'pipeline_id': pipeline_id, } status_tree = pipeline.get_status_tree(pipeline_id) if not status_tree: return status_dict for info in status_tree.get('pipelines', {}).itervalues(): if info.get('classPath') == 'mapreduce.mapper_pipeline.MapperPipeline': status_dict['pipeline_status'] = info['status'] if 'statusConsoleUrl' not in info: return status_dict mapreduce_id = ( info['statusConsoleUrl'][info['statusConsoleUrl'].rfind('=') + 1:]) status_dict['mapreduce_id'] = mapreduce_id mr_job = mr_model.MapreduceState.get_by_key_name(mapreduce_id) if not mr_job: return status_dict counters = mr_job.counters_map.to_json().get('counters', {}) counters = { key.replace('-', '_'): value for key, value in counters.iteritems() } status_dict.update({ 'mapreduce_counters': counters, 'mapreduce_active': mr_job.active, 'mapreduce_result_status': mr_job.result_status, }) return status_dict return status_dict
def get_status(pipeline_id): """Hack into the pipelines models to gather pipeline and mapreduce details.""" status_dict = { 'pipeline_id': pipeline_id, } status_tree = pipeline.get_status_tree(pipeline_id) if not status_tree: return status_dict for info in status_tree.get('pipelines', {}).itervalues(): if info.get('classPath') == 'mapreduce.mapper_pipeline.MapperPipeline': status_dict['pipeline_status'] = info['status'] if 'statusConsoleUrl' not in info: return status_dict mapreduce_id = ( info['statusConsoleUrl'][info['statusConsoleUrl'].rfind('=')+1:]) status_dict['mapreduce_id'] = mapreduce_id mr_job = mr_model.MapreduceState.get_by_key_name(mapreduce_id) if not mr_job: return status_dict counters = mr_job.counters_map.to_json().get('counters', {}) counters = {key.replace('-', '_'): value for key, value in counters.iteritems()} status_dict.update({ 'mapreduce_counters': counters, 'mapreduce_active': mr_job.active, 'mapreduce_result_status': mr_job.result_status, }) return status_dict return status_dict
def get(self): id = self.request.get("id") pipeline_id = self.request.get("root_pipeline_id") assert id or pipeline_id if id: task = _TA_Task.get_by_id(id) assert task and task.root_pipeline_id root_pipeline_id = task.root_pipeline_id else: root_pipeline_id = pipeline_id r = pipeline.get_status_tree(root_pipeline_id) assert 'pipelines' in r and root_pipeline_id in r['pipelines'] root_pipeline = r['pipelines'][root_pipeline_id] outputs = root_pipeline["outputs"] outputs = zip(outputs.keys(), db.get(outputs.values())) outputs = {k[0]: k[1].value if k[1].status == pipeline_models._SlotRecord.FILLED else None for k in outputs} assert root_pipeline['status'] in self.STATUS_MAP self.output({ "id": id, "status": self.STATUS_MAP[root_pipeline['status']], "msg": root_pipeline.get('lastRetryMessage'), "output": outputs })
def handle(self): import pipeline # Break circular dependency try: depth = int(self.request.get('depth')) except ValueError: depth = None self.json_response.update( pipeline.get_status_tree(self.request.get('root_pipeline_id'), depth))
def handle(self): import pipeline # Break circular dependency try: depth = int(self.request.get('depth')) except ValueError: logging.debug("depth %r is non-integer, assuming no depth restriction", self.request.get('depth')) depth = None self.json_response.update( pipeline.get_status_tree(self.request.get('root_pipeline_id'), depth))
def handle(self): import pipeline # Break circular dependency self.json_response.update( pipeline.get_status_tree(self.request.get('root_pipeline_id')))