def _get_mapper_spec(self):
"""Converts self to model.MapperSpec."""
from google.appengine.ext.mapreduce import model
return model.MapperSpec(
handler_spec=util._obj_to_path(self.mapper),
input_reader_spec=util._obj_to_path(self.input_reader_cls),
params=self._get_mapper_params(),
shard_count=self.shard_count,
output_writer_spec=util._obj_to_path(self.output_writer_cls))
def _get_mapper_spec(self):
"""Converts self to model.MapperSpec."""
from google.appengine.ext.mapreduce import model
return model.MapperSpec(
handler_spec=util._obj_to_path(self.mapper),
input_reader_spec=util._obj_to_path(self.input_reader_cls),
params=self._get_mapper_params(),
shard_count=self.shard_count,
output_writer_spec=util._obj_to_path(self.output_writer_cls))
def submit(cls, job_config, in_xg_transaction=False):
"""Submit the job to run.
Args:
job_config: an instance of map_job.MapJobConfig.
in_xg_transaction: controls what transaction scope to use to start this MR
job. If True, there has to be an already opened cross-group transaction
scope. MR will use one entity group from it.
If False, MR will create an independent transaction to start the job
regardless of any existing transaction scopes.
Returns:
a Job instance representing the submitted job.
"""
cls.__validate_job_config(job_config)
mapper_spec = job_config._get_mapper_spec()
mapreduce_params = job_config._get_mr_params()
mapreduce_spec = model.MapreduceSpec(
job_config.job_name,
job_config.job_id,
mapper_spec.to_json(),
mapreduce_params,
util._obj_to_path(job_config._hooks_cls))
if in_xg_transaction:
propagation = db.MANDATORY
else:
propagation = db.INDEPENDENT
state = None
@db.transactional(propagation=propagation)
def _txn():
state = cls.__create_and_save_state(job_config, mapreduce_spec)
cls.__add_kickoff_task(job_config, mapreduce_spec)
return state
state = _txn()
return cls(state)
def submit(cls, job_config, in_xg_transaction=False):
"""Submit the job to run.
Args:
job_config: an instance of map_job.MapJobConfig.
in_xg_transaction: controls what transaction scope to use to start this MR
job. If True, there has to be an already opened cross-group transaction
scope. MR will use one entity group from it.
If False, MR will create an independent transaction to start the job
regardless of any existing transaction scopes.
Returns:
a Job instance representing the submitted job.
"""
cls.__validate_job_config(job_config)
mapper_spec = job_config._get_mapper_spec()
mapreduce_params = job_config._get_mr_params()
mapreduce_spec = model.MapreduceSpec(
job_config.job_name, job_config.job_id, mapper_spec.to_json(),
mapreduce_params, util._obj_to_path(job_config._hooks_cls))
if in_xg_transaction:
propagation = db.MANDATORY
else:
propagation = db.INDEPENDENT
state = None
@db.transactional(propagation=propagation)
def _txn():
state = cls.__create_and_save_state(job_config, mapreduce_spec)
cls.__add_kickoff_task(job_config, mapreduce_spec)
return state
state = _txn()
return cls(state)