Example #1
0
def get_status(pipeline_id):
    """Hack into the pipelines models to gather pipeline and mapreduce details."""

    status_dict = {
        'pipeline_id': pipeline_id,
    }
    status_tree = pipeline.get_status_tree(pipeline_id)
    if not status_tree:
        return status_dict
    for info in status_tree.get('pipelines', {}).itervalues():
        if info.get('classPath') == 'mapreduce.mapper_pipeline.MapperPipeline':
            status_dict['pipeline_status'] = info['status']
            if 'statusConsoleUrl' not in info:
                return status_dict
            mapreduce_id = (
                info['statusConsoleUrl'][info['statusConsoleUrl'].rfind('=') +
                                         1:])
            status_dict['mapreduce_id'] = mapreduce_id
            mr_job = mr_model.MapreduceState.get_by_key_name(mapreduce_id)
            if not mr_job:
                return status_dict
            counters = mr_job.counters_map.to_json().get('counters', {})
            counters = {
                key.replace('-', '_'): value
                for key, value in counters.iteritems()
            }
            status_dict.update({
                'mapreduce_counters': counters,
                'mapreduce_active': mr_job.active,
                'mapreduce_result_status': mr_job.result_status,
            })
            return status_dict
    return status_dict
def get_status(pipeline_id):
  """Hack into the pipelines models to gather pipeline and mapreduce details."""

  status_dict = {
    'pipeline_id': pipeline_id,
  }
  status_tree = pipeline.get_status_tree(pipeline_id)
  if not status_tree:
    return status_dict
  for info in status_tree.get('pipelines', {}).itervalues():
    if info.get('classPath') == 'mapreduce.mapper_pipeline.MapperPipeline':
      status_dict['pipeline_status'] = info['status']
      if 'statusConsoleUrl' not in info:
        return status_dict
      mapreduce_id = (
          info['statusConsoleUrl'][info['statusConsoleUrl'].rfind('=')+1:])
      status_dict['mapreduce_id'] = mapreduce_id
      mr_job = mr_model.MapreduceState.get_by_key_name(mapreduce_id)
      if not mr_job:
        return status_dict
      counters = mr_job.counters_map.to_json().get('counters', {})
      counters = {key.replace('-', '_'): value
                  for key, value in counters.iteritems()}
      status_dict.update({
        'mapreduce_counters': counters,
        'mapreduce_active': mr_job.active,
        'mapreduce_result_status': mr_job.result_status,
      })
      return status_dict
  return status_dict
Example #3
0
    def get(self):
        id = self.request.get("id")
        pipeline_id = self.request.get("root_pipeline_id")
        assert id or pipeline_id

        if id:
            task = _TA_Task.get_by_id(id)
            assert task and task.root_pipeline_id
            root_pipeline_id = task.root_pipeline_id
        else:
            root_pipeline_id = pipeline_id

        r = pipeline.get_status_tree(root_pipeline_id)
        assert 'pipelines' in r and root_pipeline_id in r['pipelines']

        root_pipeline = r['pipelines'][root_pipeline_id]
        outputs = root_pipeline["outputs"]
        outputs = zip(outputs.keys(), db.get(outputs.values()))
        outputs = {k[0]: k[1].value if k[1].status == pipeline_models._SlotRecord.FILLED else None for k in outputs}

        assert root_pipeline['status'] in self.STATUS_MAP

        self.output({
            "id": id,
            "status": self.STATUS_MAP[root_pipeline['status']],
            "msg": root_pipeline.get('lastRetryMessage'),
            "output": outputs
        })
Example #4
0
 def handle(self):
   import pipeline  # Break circular dependency
   try:
     depth = int(self.request.get('depth'))
   except ValueError:
     depth = None
   self.json_response.update(
       pipeline.get_status_tree(self.request.get('root_pipeline_id'), depth))
Example #5
0
 def handle(self):
     import pipeline  # Break circular dependency
     try:
         depth = int(self.request.get('depth'))
     except ValueError:
         depth = None
     self.json_response.update(
         pipeline.get_status_tree(self.request.get('root_pipeline_id'),
                                  depth))
Example #6
0
 def handle(self):
   import pipeline  # Break circular dependency
   try:
     depth = int(self.request.get('depth'))
   except ValueError:
     logging.debug("depth %r is non-integer, assuming no depth restriction",
                   self.request.get('depth'))
     depth = None
   self.json_response.update(
       pipeline.get_status_tree(self.request.get('root_pipeline_id'), depth))
Example #7
0
 def handle(self):
   import pipeline  # Break circular dependency
   self.json_response.update(
       pipeline.get_status_tree(self.request.get('root_pipeline_id')))
Example #8
0
 def handle(self):
   import pipeline  # Break circular dependency
   self.json_response.update(
       pipeline.get_status_tree(self.request.get('root_pipeline_id')))