예제 #1
0
  def validate_map_started(self):
    # Only one kickoff task.
    tasks = self.taskqueue.GetTasks(self.config.queue_name)
    self.assertEqual(1, len(tasks))
    self.taskqueue.FlushQueue(self.config.queue_name)
    # Hook was run.
    self.assertEqual(1, len(TestHooks.enqueue_kickoff_task_calls))

    # Check the task.
    task = tasks[0]
    self.assertTrue(task["url"].startswith(self.config._base_path))
    # Check task header.
    headers = dict(task["headers"])
    self.assertEqual(self.config.job_id, headers[util._MR_ID_TASK_HEADER])
    # Check task payload.
    task_mr_id = test_support.decode_task_payload(task).get("mapreduce_id")
    self.assertEqual(self.config.job_id, task_mr_id)

    # Check state.
    state = model.MapreduceState.get_by_job_id(self.config.job_id)
    self.assertTrue(state.active)
    self.assertEqual(0, state.active_shards)

    test_support.execute_task(task)

    # controller + shard tasks.
    tasks = self.taskqueue.GetTasks(self.config.queue_name)
    self.assertEqual(1 + self.config.shard_count, len(tasks))

    state = model.MapreduceState.get_by_job_id(self.config.job_id)
    self.assertEqual("__main__.TestHooks",
                     state.mapreduce_spec.hooks_class_name)

    # Verify mapreduce_spec.mapper_spec
    mapper_spec = state.mapreduce_spec.mapper
    self.assertEqual("mapreduce.api.map_job."
                     "mapper.Mapper",
                     mapper_spec.handler_spec)
    self.assertEqual("mapreduce.api.map_job."
                     "sample_input_reader.SampleInputReader",
                     mapper_spec.input_reader_spec)
    self.assertEqual(self.config.input_reader_params,
                     {"count": TEST_SAMPLE_INPUT_READER_COUNT})
    # Verify mapreduce_spec.params.
    self.assertEqual(self.config.queue_name,
                     state.mapreduce_spec.params["queue_name"])
    self.assertEqual(self.config._force_writes,
                     state.mapreduce_spec.params["force_writes"])
    self.assertEqual(self.config.done_callback_url,
                     state.mapreduce_spec.params["done_callback"])
    self.assertEqual(self.config._base_path,
                     state.mapreduce_spec.params["base_path"])
    self.assertEqual(self.config.shard_max_attempts,
                     state.mapreduce_spec.params["shard_max_attempts"])
    self.assertEqual(self.config._task_max_attempts,
                     state.mapreduce_spec.params["task_max_attempts"])
    self.assertEqual(self.config._api_version,
                     state.mapreduce_spec.params["api_version"])
    self.assertEqual(self.config._api_version, map_job_config._API_VERSION)
 def get_mapreduce_spec(self, task):
     """Get mapreduce spec form kickoff task payload."""
     payload = test_support.decode_task_payload(task)
     return model.MapreduceSpec.from_json_str(payload["mapreduce_spec"])
예제 #3
0
 def get_mapreduce_spec(self, task):
   """Get mapreduce spec form kickoff task payload."""
   payload = test_support.decode_task_payload(task)
   return model.MapreduceSpec.from_json_str(payload["mapreduce_spec"])