def _get_mapper_spec(self):
"""Converts self to model.MapperSpec."""
return model.MapperSpec(
handler_spec=util._obj_to_path(self.mapper),
input_reader_spec=util._obj_to_path(self.input_reader_cls),
params=self._get_mapper_params(),
shard_count=self.shard_count,
output_writer_spec=util._obj_to_path(self.output_writer_cls))
def _get_mapper_spec(self):
"""Converts self to model.MapperSpec."""
# pylint: disable=g-import-not-at-top
from mapreduce import model
return model.MapperSpec(
handler_spec=util._obj_to_path(self.mapper),
input_reader_spec=util._obj_to_path(self.input_reader_cls),
params=self._get_mapper_params(),
shard_count=self.shard_count,
output_writer_spec=util._obj_to_path(self.output_writer_cls))
def start(job_config=None,
in_xg_transaction=False):
"""Start a new map job.
Args:
job_config: an instance of map_job.MapJobConfig.
in_xg_transaction: controls what transaction scope to use to start this MR
job. If True, there has to be an already opened cross-group transaction
scope. MR will use one entity group from it.
If False, MR will create an independent transaction to start the job
regardless of any existing transaction scopes.
Returns:
the id of this map job.
Raises:
ValueError: when in_xg_transaction is True but no transaction scope is
detected.
"""
if in_xg_transaction and not db.is_in_transaction():
raise ValueError("Expects an opened xg transaction to start mapreduce.")
# Break circular dependency.
# pylint: disable=g-import-not-at-top
from mapreduce import handlers
return handlers.StartJobHandler._start_map(
name=job_config.job_name,
mapper_spec=job_config._get_mapper_spec(),
mapreduce_params=job_config._get_mr_params(),
queue_name=job_config.queue_name,
hooks_class_name=util._obj_to_path(job_config._hooks_cls),
_app=job_config._app,
in_xg_transaction=in_xg_transaction)
def submit(cls, job_config, in_xg_transaction=False):
"""Submit the job to run.
Args:
job_config: an instance of map_job.MapJobConfig.
in_xg_transaction: controls what transaction scope to use to start this MR
job. If True, there has to be an already opened cross-group transaction
scope. MR will use one entity group from it.
If False, MR will create an independent transaction to start the job
regardless of any existing transaction scopes.
Returns:
a Job instance representing the submitted job.
"""
cls.__validate_job_config(job_config)
mapper_spec = job_config._get_mapper_spec()
# Create mr spec.
mapreduce_params = job_config._get_mr_params()
mapreduce_spec = model.MapreduceSpec(
job_config.job_name,
job_config.job_id,
mapper_spec.to_json(),
mapreduce_params,
util._obj_to_path(job_config._hooks_cls))
# Save states and enqueue task.
if in_xg_transaction:
propagation = db.MANDATORY
else:
propagation = db.INDEPENDENT
state = None
@db.transactional(propagation=propagation)
def _txn():
state = cls.__create_and_save_state(job_config, mapreduce_spec)
cls.__add_kickoff_task(job_config, mapreduce_spec)
return state
state = _txn()
return cls(state)
def submit(cls, job_config, in_xg_transaction=False):
"""Submit the job to run.
Args:
job_config: an instance of map_job.MapJobConfig.
in_xg_transaction: controls what transaction scope to use to start this MR
job. If True, there has to be an already opened cross-group transaction
scope. MR will use one entity group from it.
If False, MR will create an independent transaction to start the job
regardless of any existing transaction scopes.
Returns:
a Job instance representing the submitted job.
"""
cls.__validate_job_config(job_config)
mapper_spec = job_config._get_mapper_spec()
# Create mr spec.
mapreduce_params = job_config._get_mr_params()
mapreduce_spec = model.MapreduceSpec(
job_config.job_name, job_config.job_id, mapper_spec.to_json(),
mapreduce_params, util._obj_to_path(job_config._hooks_cls))
# Save states and enqueue task.
if in_xg_transaction:
propagation = db.MANDATORY
else:
propagation = db.INDEPENDENT
state = None
@db.transactional(propagation=propagation)
def _txn():
state = cls.__create_and_save_state(job_config, mapreduce_spec)
cls.__add_kickoff_task(job_config, mapreduce_spec)
return state
state = _txn()
return cls(state)
def start(job_config=None,
in_xg_transaction=False):
"""Start a new map job.
Args:
job_config: an instance of map_job.MapJobConfig.
in_xg_transaction: controls what transaction scope to use to start this MR
job. If True, there has to be an already opened cross-group transaction
scope. MR will use one entity group from it.
If False, MR will create an independent transaction to start the job
regardless of any existing transaction scopes.
"""
# Validate input reader and output writer.
mapper_spec = job_config._get_mapper_spec()
job_config.input_reader_cls.validate(mapper_spec)
if job_config.output_writer_cls:
job_config.output_writer_cls.validate(mapper_spec)
# Create mr spec.
mapreduce_params = job_config._get_mr_params()
mapreduce_spec = model.MapreduceSpec(
job_config.job_name,
job_config.job_id,
mapper_spec.to_json(),
mapreduce_params,
util._obj_to_path(job_config._hooks_cls))
# Save states and enqueue task.
if in_xg_transaction:
propagation = db.MANDATORY
else:
propagation = db.INDEPENDENT
@db.transactional(propagation=propagation)
def _txn():
_create_and_save_state(job_config, mapreduce_spec)
_add_kickoff_task(job_config, mapreduce_spec)
_txn()
def testBasic(self):
self.assertEqual(None, util._obj_to_path(None))
self.assertEqual("__main__.FooClass", util._obj_to_path(FooClass))
self.assertEqual("__main__.test_handler_function",
util._obj_to_path(test_handler_function))
def testBasic(self):
self.assertEqual(None, util._obj_to_path(None))
self.assertEqual("__main__.FooClass", util._obj_to_path(FooClass))
self.assertEqual("__main__.test_handler_function",
util._obj_to_path(test_handler_function))