Python _obj_to_path Examples

Example #1

File: map_job_config.py Project: startup-one/cogofly

    def _get_mapper_spec(self):
        """Converts self to model.MapperSpec."""

        from google.appengine.ext.mapreduce import model

        return model.MapperSpec(
            handler_spec=util._obj_to_path(self.mapper),
            input_reader_spec=util._obj_to_path(self.input_reader_cls),
            params=self._get_mapper_params(),
            shard_count=self.shard_count,
            output_writer_spec=util._obj_to_path(self.output_writer_cls))

Example #2

File: map_job_config.py Project: 404minds/quiz-forest

  def _get_mapper_spec(self):
    """Converts self to model.MapperSpec."""

    from google.appengine.ext.mapreduce import model

    return model.MapperSpec(
        handler_spec=util._obj_to_path(self.mapper),
        input_reader_spec=util._obj_to_path(self.input_reader_cls),
        params=self._get_mapper_params(),
        shard_count=self.shard_count,
        output_writer_spec=util._obj_to_path(self.output_writer_cls))

Example #3

File: map_job_control.py Project: 404minds/quiz-forest

  def submit(cls, job_config, in_xg_transaction=False):
    """Submit the job to run.

    Args:
      job_config: an instance of map_job.MapJobConfig.
      in_xg_transaction: controls what transaction scope to use to start this MR
        job. If True, there has to be an already opened cross-group transaction
        scope. MR will use one entity group from it.
        If False, MR will create an independent transaction to start the job
        regardless of any existing transaction scopes.

    Returns:
      a Job instance representing the submitted job.
    """
    cls.__validate_job_config(job_config)
    mapper_spec = job_config._get_mapper_spec()


    mapreduce_params = job_config._get_mr_params()
    mapreduce_spec = model.MapreduceSpec(
        job_config.job_name,
        job_config.job_id,
        mapper_spec.to_json(),
        mapreduce_params,
        util._obj_to_path(job_config._hooks_cls))


    if in_xg_transaction:
      propagation = db.MANDATORY
    else:
      propagation = db.INDEPENDENT

    state = None
    @db.transactional(propagation=propagation)
    def _txn():
      state = cls.__create_and_save_state(job_config, mapreduce_spec)
      cls.__add_kickoff_task(job_config, mapreduce_spec)
      return state

    state = _txn()
    return cls(state)

Example #4

File: map_job_control.py Project: Dean-Christian-Armada/django

    def submit(cls, job_config, in_xg_transaction=False):
        """Submit the job to run.

    Args:
      job_config: an instance of map_job.MapJobConfig.
      in_xg_transaction: controls what transaction scope to use to start this MR
        job. If True, there has to be an already opened cross-group transaction
        scope. MR will use one entity group from it.
        If False, MR will create an independent transaction to start the job
        regardless of any existing transaction scopes.

    Returns:
      a Job instance representing the submitted job.
    """
        cls.__validate_job_config(job_config)
        mapper_spec = job_config._get_mapper_spec()

        mapreduce_params = job_config._get_mr_params()
        mapreduce_spec = model.MapreduceSpec(
            job_config.job_name, job_config.job_id, mapper_spec.to_json(),
            mapreduce_params, util._obj_to_path(job_config._hooks_cls))

        if in_xg_transaction:
            propagation = db.MANDATORY
        else:
            propagation = db.INDEPENDENT

        state = None

        @db.transactional(propagation=propagation)
        def _txn():
            state = cls.__create_and_save_state(job_config, mapreduce_spec)
            cls.__add_kickoff_task(job_config, mapreduce_spec)
            return state

        state = _txn()
        return cls(state)