def AbortAdminOperation(operation_key, _status=DatastoreAdminOperation.STATUS_ABORTED):
"""Aborts active jobs."""
operation = DatastoreAdminOperation.get(operation_key)
operation.status = _status
operation.put(config=_CreateDatastoreConfig())
for job in operation.active_job_ids:
logging.info("Aborting Job %s", job)
model.MapreduceControl.abort(job, config=_CreateDatastoreConfig())
def AbortAdminOperation(operation_key,
_status=DatastoreAdminOperation.STATUS_ABORTED):
"""Aborts active jobs."""
operation = DatastoreAdminOperation.get(operation_key)
operation.status = _status
operation.put(config=_CreateDatastoreConfig())
for job in operation.active_job_ids:
logging.info('Aborting Job %s', job)
model.MapreduceControl.abort(job, config=_CreateDatastoreConfig())
def StartOperation(description):
"""Start datastore admin operation.
Args:
description: operation description to be displayed to user.
Returns:
an instance of DatastoreAdminOperation.
"""
operation = DatastoreAdminOperation(
description=description, id=db.allocate_ids(db.Key.from_path(DatastoreAdminOperation.kind(), 1), 1)[0]
)
operation.put(config=_CreateDatastoreConfig())
return operation
def StartOperation(description):
"""Start datastore admin operation.
Args:
description: operation description to be displayed to user.
Returns:
an instance of DatastoreAdminOperation.
"""
operation = DatastoreAdminOperation(
description=description,
id=db.allocate_ids(db.Key.from_path(DatastoreAdminOperation.kind(), 1),
1)[0])
operation.put(config=_CreateDatastoreConfig())
return operation
def tx():
operation = DatastoreAdminOperation.get(operation_key)
job_id = control.start_map(
job_name, handler_spec, reader_spec,
mapper_params,
output_writer_spec=writer_spec,
mapreduce_parameters=mapreduce_params,
base_path=config.MAPREDUCE_PATH,
shard_count=shard_count,
transactional=True,
queue_name=queue_name,
transactional_parent=operation)
operation.status = DatastoreAdminOperation.STATUS_ACTIVE
operation.active_jobs += 1
operation.active_job_ids = list(set(operation.active_job_ids + [job_id]))
operation.put(config=_CreateDatastoreConfig())
return job_id
def tx():
operation = DatastoreAdminOperation.get(operation_key)
job_id = control.start_map(job_name,
handler_spec,
reader_spec,
mapper_params,
output_writer_spec=writer_spec,
mapreduce_parameters=mapreduce_params,
base_path=config.MAPREDUCE_PATH,
shard_count=shard_count,
transactional=True,
queue_name=queue_name,
transactional_parent=operation)
operation.status = DatastoreAdminOperation.STATUS_ACTIVE
operation.active_jobs += 1
operation.active_job_ids = list(
set(operation.active_job_ids + [job_id]))
operation.put(config=_CreateDatastoreConfig())
return job_id
def tx():
operation = DatastoreAdminOperation.get(operation_key)
if mapreduce_id in operation.active_job_ids:
operation.active_jobs -= 1
operation.completed_jobs += 1
operation.active_job_ids.remove(mapreduce_id)
if not operation.active_jobs:
if operation.status == DatastoreAdminOperation.STATUS_ACTIVE:
operation.status = DatastoreAdminOperation.STATUS_COMPLETED
db.delete(DatastoreAdminOperationJob.all().ancestor(operation),
config=db_config)
operation.put(config=db_config)
if 'done_callback_handler' in mapreduce_params:
done_callback_handler = util.for_name(
mapreduce_params['done_callback_handler'])
if done_callback_handler:
done_callback_handler(operation, mapreduce_id, mapreduce_state)
else:
logging.error('done_callbackup_handler %s was not found',
mapreduce_params['done_callback_handler'])
def tx():
operation = DatastoreAdminOperation.get(operation_key)
if mapreduce_id in operation.active_job_ids:
operation.active_jobs -= 1
operation.completed_jobs += 1
operation.active_job_ids.remove(mapreduce_id)
if not operation.active_jobs:
if operation.status == DatastoreAdminOperation.STATUS_ACTIVE:
operation.status = DatastoreAdminOperation.STATUS_COMPLETED
db.delete(
DatastoreAdminOperationJob.all().ancestor(
operation),
config=db_config)
operation.put(config=db_config)
if 'done_callback_handler' in mapreduce_params:
done_callback_handler = util.for_name(
mapreduce_params['done_callback_handler'])
if done_callback_handler:
done_callback_handler(operation, mapreduce_id,
mapreduce_state)
else:
logging.error(
'done_callbackup_handler %s was not found',
mapreduce_params['done_callback_handler'])