def _get_mapper_spec(self):
   """Converts self to model.MapperSpec."""
   return model.MapperSpec(
       handler_spec=util._obj_to_path(self.mapper),
       input_reader_spec=util._obj_to_path(self.input_reader_cls),
       params=self._get_mapper_params(),
       shard_count=self.shard_count,
       output_writer_spec=util._obj_to_path(self.output_writer_cls))
  def _get_mapper_spec(self):
    """Converts self to model.MapperSpec."""
    # pylint: disable=g-import-not-at-top
    from mapreduce import model

    return model.MapperSpec(
        handler_spec=util._obj_to_path(self.mapper),
        input_reader_spec=util._obj_to_path(self.input_reader_cls),
        params=self._get_mapper_params(),
        shard_count=self.shard_count,
        output_writer_spec=util._obj_to_path(self.output_writer_cls))
def start(job_config=None,
          in_xg_transaction=False):
  """Start a new map job.

  Args:
    job_config: an instance of map_job.MapJobConfig.
    in_xg_transaction: controls what transaction scope to use to start this MR
      job. If True, there has to be an already opened cross-group transaction
      scope. MR will use one entity group from it.
      If False, MR will create an independent transaction to start the job
      regardless of any existing transaction scopes.

  Returns:
    the id of this map job.

  Raises:
    ValueError: when in_xg_transaction is True but no transaction scope is
      detected.
  """
  if in_xg_transaction and not db.is_in_transaction():
    raise ValueError("Expects an opened xg transaction to start mapreduce.")

  # Break circular dependency.
  # pylint: disable=g-import-not-at-top
  from mapreduce import handlers

  return handlers.StartJobHandler._start_map(
      name=job_config.job_name,
      mapper_spec=job_config._get_mapper_spec(),
      mapreduce_params=job_config._get_mr_params(),
      queue_name=job_config.queue_name,
      hooks_class_name=util._obj_to_path(job_config._hooks_cls),
      _app=job_config._app,
      in_xg_transaction=in_xg_transaction)
示例#4
0
  def submit(cls, job_config, in_xg_transaction=False):
    """Submit the job to run.

    Args:
      job_config: an instance of map_job.MapJobConfig.
      in_xg_transaction: controls what transaction scope to use to start this MR
        job. If True, there has to be an already opened cross-group transaction
        scope. MR will use one entity group from it.
        If False, MR will create an independent transaction to start the job
        regardless of any existing transaction scopes.

    Returns:
      a Job instance representing the submitted job.
    """
    cls.__validate_job_config(job_config)
    mapper_spec = job_config._get_mapper_spec()

    # Create mr spec.
    mapreduce_params = job_config._get_mr_params()
    mapreduce_spec = model.MapreduceSpec(
        job_config.job_name,
        job_config.job_id,
        mapper_spec.to_json(),
        mapreduce_params,
        util._obj_to_path(job_config._hooks_cls))

    # Save states and enqueue task.
    if in_xg_transaction:
      propagation = db.MANDATORY
    else:
      propagation = db.INDEPENDENT

    state = None
    @db.transactional(propagation=propagation)
    def _txn():
      state = cls.__create_and_save_state(job_config, mapreduce_spec)
      cls.__add_kickoff_task(job_config, mapreduce_spec)
      return state

    state = _txn()
    return cls(state)
    def submit(cls, job_config, in_xg_transaction=False):
        """Submit the job to run.

    Args:
      job_config: an instance of map_job.MapJobConfig.
      in_xg_transaction: controls what transaction scope to use to start this MR
        job. If True, there has to be an already opened cross-group transaction
        scope. MR will use one entity group from it.
        If False, MR will create an independent transaction to start the job
        regardless of any existing transaction scopes.

    Returns:
      a Job instance representing the submitted job.
    """
        cls.__validate_job_config(job_config)
        mapper_spec = job_config._get_mapper_spec()

        # Create mr spec.
        mapreduce_params = job_config._get_mr_params()
        mapreduce_spec = model.MapreduceSpec(
            job_config.job_name, job_config.job_id, mapper_spec.to_json(),
            mapreduce_params, util._obj_to_path(job_config._hooks_cls))

        # Save states and enqueue task.
        if in_xg_transaction:
            propagation = db.MANDATORY
        else:
            propagation = db.INDEPENDENT

        state = None

        @db.transactional(propagation=propagation)
        def _txn():
            state = cls.__create_and_save_state(job_config, mapreduce_spec)
            cls.__add_kickoff_task(job_config, mapreduce_spec)
            return state

        state = _txn()
        return cls(state)
def start(job_config=None,
          in_xg_transaction=False):
  """Start a new map job.

  Args:
    job_config: an instance of map_job.MapJobConfig.
    in_xg_transaction: controls what transaction scope to use to start this MR
      job. If True, there has to be an already opened cross-group transaction
      scope. MR will use one entity group from it.
      If False, MR will create an independent transaction to start the job
      regardless of any existing transaction scopes.
  """
  # Validate input reader and output writer.
  mapper_spec = job_config._get_mapper_spec()
  job_config.input_reader_cls.validate(mapper_spec)
  if job_config.output_writer_cls:
    job_config.output_writer_cls.validate(mapper_spec)

  # Create mr spec.
  mapreduce_params = job_config._get_mr_params()
  mapreduce_spec = model.MapreduceSpec(
      job_config.job_name,
      job_config.job_id,
      mapper_spec.to_json(),
      mapreduce_params,
      util._obj_to_path(job_config._hooks_cls))

  # Save states and enqueue task.
  if in_xg_transaction:
    propagation = db.MANDATORY
  else:
    propagation = db.INDEPENDENT

  @db.transactional(propagation=propagation)
  def _txn():
    _create_and_save_state(job_config, mapreduce_spec)
    _add_kickoff_task(job_config, mapreduce_spec)

  _txn()
示例#7
0
 def testBasic(self):
     self.assertEqual(None, util._obj_to_path(None))
     self.assertEqual("__main__.FooClass", util._obj_to_path(FooClass))
     self.assertEqual("__main__.test_handler_function",
                      util._obj_to_path(test_handler_function))
 def testBasic(self):
   self.assertEqual(None, util._obj_to_path(None))
   self.assertEqual("__main__.FooClass", util._obj_to_path(FooClass))
   self.assertEqual("__main__.test_handler_function",
                    util._obj_to_path(test_handler_function))