def testSuccessfulJob(self):
"""Verify that with appropriate request parameters form is constructed."""
TestEntity().put()
admin_operation = utils.StartOperation("Test Operation")
self.mapreduce_id = utils.StartMap(
admin_operation,
'test_job',
'__main__.foo',
('google.appengine.ext.mapreduce.input_readers.'
'DatastoreKeyInputReader'),
{'entity_kind': 'TestEntity'})
testutil.execute_all_tasks(self.taskqueue)
self.assertObjectsExist()
testutil.execute_until_empty(self.taskqueue)
self.handler.request.headers['Mapreduce-Id'] = self.mapreduce_id
self.handler.post()
self.assertIsNone(model.MapreduceState.get_by_key_name(self.mapreduce_id))
self.assertListEqual(
[],
model.ShardState.find_by_mapreduce_id(self.mapreduce_id))
admin_operation = admin_operation.get(admin_operation.key())
self.assertEqual(0, admin_operation.active_jobs)
self.assertEqual(1, admin_operation.completed_jobs)
self.assertEqual('Completed', admin_operation.status)
def _ProcessPostRequest(self):
"""Triggers backup restore mapper jobs and returns their ids."""
backup_id = self.request.get('backup_id')
if not backup_id:
return [('error', 'Unspecified Backup.')]
backup = db.get(db.Key(backup_id))
if not backup:
return [('error', 'Invalid Backup id.')]
queue = self.request.get('queue')
job_name = 'datastore_backup_restore_%s' % re.sub(r'[^\w]', '_',
backup.name)
job_operation = None
kinds = set(self.request.get_all('kind'))
if not (backup.blob_files or kinds):
return [('error', 'No kinds were selected')]
backup_kinds = set(backup.kinds)
difference = kinds.difference(backup_kinds)
if difference:
return [('error', 'Backup does not have kind[s] %s' %
', '.join(difference))]
kinds = list(kinds) if len(backup_kinds) != len(kinds) else []
try:
operation_name = 'Restoring %s from backup: %s' % (
', '.join(kinds) if kinds else 'all', backup.name)
job_operation = utils.StartOperation(operation_name)
mapper_params = self._GetBasicMapperParams()
mapper_params['files'] = get_backup_files(backup, kinds)
mapper_params['kind_filter'] = kinds
mapper_params['original_app'] = backup.original_app
mapreduce_params = {
'backup_name': backup.name,
'force_ops_writes': True
}
shard_count = min(max(utils.MAPREDUCE_MIN_SHARDS,
len(mapper_params['files'])),
utils.MAPREDUCE_MAX_SHARDS)
job = utils.StartMap(job_operation.key(), job_name,
self.BACKUP_RESTORE_HANDLER, self.INPUT_READER, None,
mapper_params, mapreduce_params, queue_name=queue,
shard_count=shard_count)
return [('job', job)]
except Exception:
logging.exception('Failed to start a restore from backup job "%s".',
job_name)
if job_operation:
job_operation.status = utils.DatastoreAdminOperation.STATUS_FAILED
job_operation.put(force_writes=True)
raise
finally:
if self.request.get('delete_backup_after_restore', '').lower() == 'true':
delete_backup_info(backup, delete_files=False)
def _ProcessPostRequest(self):
"""Triggers backup restore mapper jobs and returns their ids."""
backup_id = self.request.get('backup_id')
if not backup_id:
return [('error', 'Unspecified Backup.')]
backup = db.get(db.Key(backup_id))
if not backup:
return [('error', 'Invalid Backup id.')]
queue = self.request.get('queue')
job_name = 'datastore_backup_%s' % re.sub(r'[^\w]', '_', backup.name)
job_operation = None
try:
job_operation = utils.StartOperation('Restore from backup: %s' %
backup.name)
mapper_params = self._GetBasicMapperParams()
mapper_params['files'] = backup.blob_files
mapreduce_params = {
'backup_name': backup.name,
'force_ops_writes': True
}
return [('job',
utils.StartMap(job_operation.key(),
job_name,
self.BACKUP_RESTORE_HANDLER,
self.INPUT_READER,
None,
mapper_params,
mapreduce_params,
queue_name=queue))]
except Exception:
logging.exception(
'Failed to start a restore from backup job "%s".', job_name)
if job_operation:
job_operation.status = utils.DatastoreAdminOperation.STATUS_FAILED
job_operation.put(config=datastore_rpc.Configuration(
force_writes=True))
raise