def _schedule_shards(cls, spec, input_readers, output_writers, queue_name,
base_path):
"""Prepares shard states and schedules their execution.
Args:
spec: mapreduce specification as MapreduceSpec.
input_readers: list of InputReaders describing shard splits.
queue_name: The queue to run this job on.
base_path: The base url path of mapreduce callbacks.
"""
assert len(input_readers) == len(output_writers)
# Note: it's safe to re-attempt this handler because:
# - shard state has deterministic and unique key.
# - _schedule_slice will fall back gracefully if a task already exists.
shard_states = []
for shard_number, input_reader in enumerate(input_readers):
shard_state = model.ShardState.create_new(spec.mapreduce_id,
shard_number)
shard_state.shard_description = str(input_reader)
shard_states.append(shard_state)
# Retrievs already existing shards.
existing_shard_states = db.get(shard.key() for shard in shard_states)
existing_shard_keys = set(shard.key()
for shard in existing_shard_states
if shard is not None)
# Puts only non-existing shards.
db.put((shard for shard in shard_states
if shard.key() not in existing_shard_keys),
config=util.create_datastore_write_config(spec))
# Give each shard some quota to start with.
processing_rate = int(
spec.mapper.params.get("processing_rate")
or model._DEFAULT_PROCESSING_RATE_PER_SEC)
quota_refill = processing_rate / len(shard_states)
quota_manager = quota.QuotaManager(memcache.Client())
for shard_state in shard_states:
quota_manager.put(shard_state.shard_id, quota_refill)
# Schedule shard tasks.
for shard_number, (input_reader, output_writer) in enumerate(
zip(input_readers, output_writers)):
shard_id = model.ShardState.shard_id_from_number(
spec.mapreduce_id, shard_number)
MapperWorkerCallbackHandler._schedule_slice(
shard_states[shard_number],
model.TransientShardState(base_path,
spec,
shard_id,
0,
input_reader,
output_writer=output_writer),
queue_name=queue_name)
def _schedule_shards(cls, spec, input_readers, output_writers, queue_name,
base_path):
"""Prepares shard states and schedules their execution.
Args:
spec: mapreduce specification as MapreduceSpec.
input_readers: list of InputReaders describing shard splits.
queue_name: The queue to run this job on.
base_path: The base url path of mapreduce callbacks.
"""
assert len(input_readers) == len(output_writers)
# Note: it's safe to re-attempt this handler because:
# - shard state has deterministic and unique key.
# - _schedule_slice will fall back gracefully if a task already exists.
shard_states = []
for shard_number, input_reader in enumerate(input_readers):
shard_state = model.ShardState.create_new(spec.mapreduce_id,
shard_number)
shard_state.shard_description = str(input_reader)
shard_states.append(shard_state)
# Retrievs already existing shards.
existing_shard_states = db.get(shard.key() for shard in shard_states)
existing_shard_keys = set(shard.key()
for shard in existing_shard_states
if shard is not None)
# Puts only non-existing shards.
db.put((shard for shard in shard_states
if shard.key() not in existing_shard_keys),
config=util.create_datastore_write_config(spec))
for shard_number, (input_reader, output_writer) in enumerate(
zip(input_readers, output_writers)):
shard_id = model.ShardState.shard_id_from_number(
spec.mapreduce_id, shard_number)
MapperWorkerCallbackHandler._schedule_slice(
model.TransientShardState(base_path,
spec,
shard_id,
0,
input_reader,
output_writer=output_writer),
queue_name=queue_name)