def validate_map_started(self): # Only one kickoff task. tasks = self.taskqueue.GetTasks(self.config.queue_name) self.assertEqual(1, len(tasks)) self.taskqueue.FlushQueue(self.config.queue_name) # Hook was run. self.assertEqual(1, len(TestHooks.enqueue_kickoff_task_calls)) # Check the task. task = tasks[0] self.assertTrue(task["url"].startswith(self.config._base_path)) # Check task header. headers = dict(task["headers"]) self.assertEqual(self.config.job_id, headers[util._MR_ID_TASK_HEADER]) # Check task payload. task_mr_id = test_support.decode_task_payload(task).get("mapreduce_id") self.assertEqual(self.config.job_id, task_mr_id) # Check state. state = model.MapreduceState.get_by_job_id(self.config.job_id) self.assertTrue(state.active) self.assertEqual(0, state.active_shards) test_support.execute_task(task) # controller + shard tasks. tasks = self.taskqueue.GetTasks(self.config.queue_name) self.assertEqual(1 + self.config.shard_count, len(tasks)) state = model.MapreduceState.get_by_job_id(self.config.job_id) self.assertEqual("__main__.TestHooks", state.mapreduce_spec.hooks_class_name) # Verify mapreduce_spec.mapper_spec mapper_spec = state.mapreduce_spec.mapper self.assertEqual("mapreduce.api.map_job." "mapper.Mapper", mapper_spec.handler_spec) self.assertEqual("mapreduce.api.map_job." "sample_input_reader.SampleInputReader", mapper_spec.input_reader_spec) self.assertEqual(self.config.input_reader_params, {"count": TEST_SAMPLE_INPUT_READER_COUNT}) # Verify mapreduce_spec.params. self.assertEqual(self.config.queue_name, state.mapreduce_spec.params["queue_name"]) self.assertEqual(self.config._force_writes, state.mapreduce_spec.params["force_writes"]) self.assertEqual(self.config.done_callback_url, state.mapreduce_spec.params["done_callback"]) self.assertEqual(self.config._base_path, state.mapreduce_spec.params["base_path"]) self.assertEqual(self.config.shard_max_attempts, state.mapreduce_spec.params["shard_max_attempts"]) self.assertEqual(self.config._task_max_attempts, state.mapreduce_spec.params["task_max_attempts"]) self.assertEqual(self.config._api_version, state.mapreduce_spec.params["api_version"]) self.assertEqual(self.config._api_version, map_job_config._API_VERSION)
def get_mapreduce_spec(self, task): """Get mapreduce spec form kickoff task payload.""" payload = test_support.decode_task_payload(task) return model.MapreduceSpec.from_json_str(payload["mapreduce_spec"])
def get_mapreduce_spec(self, task): """Get mapreduce spec form kickoff task payload.""" payload = test_support.decode_task_payload(task) return model.MapreduceSpec.from_json_str(payload["mapreduce_spec"])